FEAT: Integrate mssql_py_core wheel installation into PR validation pipeline

.gitignore

@@ -61,5 +61,8 @@ build/
 *venv*/
 **/*venv*/
 
+# Extracted mssql_py_core (from eng/scripts/install-mssql-py-core)
+mssql_py_core/
+
 # learning files
 learnings/

OneBranchPipelines/jobs/consolidate-artifacts-job.yml

@@ -26,7 +26,22 @@ jobs:
         value: '$(Build.ArtifactStagingDirectory)'
 
     steps:
-      - checkout: none  # No source code needed for consolidation
+      - checkout: self  # Need source for repackaging script and version file
+        fetchDepth: 1
+
+      # Official builds must not use dev/nightly mssql-py-core versions
+      - ${{ if eq(parameters.oneBranchType, 'Official') }}:
+        - bash: |
+            set -e
+            VERSION=$(cat $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version | tr -d '[:space:]')
+            echo "mssql-py-core version: $VERSION"
+            if echo "$VERSION" | grep -qiE '(dev|nightly|alpha|beta|rc|preview)'; then
+              echo "##[error]Official builds cannot use pre-release mssql-py-core version: $VERSION"
+              echo "##[error]Update eng/versions/mssql-py-core.version to a stable release version."
+              exit 1
+            fi
+            echo "Version '$VERSION' is acceptable for Official builds."
+          displayName: 'Validate mssql-py-core version for Official build'
 
       # Download ALL artifacts from current build
       # Matrix jobs publish as: Windows_<JobId>, macOS_<JobId>, Linux_<JobId>

OneBranchPipelines/stages/build-linux-single-stage.yml

@@ -126,10 +126,10 @@ stages:
                   set -euxo pipefail
                   if command -v dnf >/dev/null 2>&1; then
                     dnf -y update || true
-                    dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true
+                    dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true
                   elif command -v yum >/dev/null 2>&1; then
                     yum -y update || true
-                    yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true
+                    yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true
                   fi
                   gcc --version || true
                   cmake --version || true
@@ -138,7 +138,7 @@ stages:
                 docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc '
                   set -euxo pipefail
                   apk update || true
-                  apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true
+                  apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true
                   gcc --version || true
                   cmake --version || true
                 '
@@ -198,6 +198,7 @@ stages:
                     PY=/opt/python/${PYBIN}-${PYBIN}/bin/python;
                     test -x $PY || { echo "Python $PY missing - skipping"; exit 0; };
                     ln -sf $PY /usr/local/bin/python;
+                    ln -sf $PY /usr/local/bin/python3;
                     echo "Using: $(python --version)";
 
                     # Step 2: Install build dependencies
@@ -208,6 +209,9 @@ stages:
                     cd /workspace/mssql_python/pybind;
                     bash build.sh;
 
+                    # Step 3.5: Extract mssql_py_core from NuGet for this Python version
+                    bash /workspace/eng/scripts/install-mssql-py-core.sh;
+
                     # Step 4: Build wheel
                     echo "Building wheel package...";
                     cd /workspace;
@@ -263,6 +267,7 @@ stages:
                     PY=/opt/python/${PYBIN}-${PYBIN}/bin/python;
                     test -x $PY || { echo "Python $PY missing - skipping"; exit 0; };
                     ln -sf $PY /usr/local/bin/python;
+                    ln -sf $PY /usr/local/bin/python3;
                     echo "Using: $(python --version)";
 
                     # Step 2: Install build dependencies
@@ -273,6 +278,9 @@ stages:
                     cd /workspace/mssql_python/pybind;
                     bash build.sh;
 
+                    # Step 3.5: Extract mssql_py_core from NuGet for this Python version
+                    bash /workspace/eng/scripts/install-mssql-py-core.sh;
+
                     # Step 4: Build wheel
                     echo "Building wheel package...";
                     cd /workspace;

OneBranchPipelines/stages/build-macos-single-stage.yml

@@ -186,6 +186,11 @@ stages:
           # =========================
           # WHEEL BUILD
           # =========================
+          # Extract mssql_py_core from NuGet into repo root before building the wheel
+          - script: |
+              bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh
+            displayName: 'Install mssql_py_core from NuGet'
+
           # Build wheel package from setup.py
           # Wheel filename: mssql_python-X.Y.Z-cp3XX-cp3XX-macosx_XX_X_universal2.whl
           # bdist_wheel = build binary wheel distribution (contains pre-compiled .so)

OneBranchPipelines/stages/build-windows-single-stage.yml

@@ -287,6 +287,13 @@ stages:
               TargetFolder: '$(Build.SourcesDirectory)\apiScan\pdbs\windows\py$(shortPyVer)\$(targetArch)'
             displayName: 'Copy PDB to ApiScan directory'
 
+          # Extract mssql_py_core from NuGet into repo root before building the wheel
+          - task: PowerShell@2
+            displayName: 'Install mssql_py_core from NuGet'
+            inputs:
+              targetType: 'filePath'
+              filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1'
+
           # Build Python wheel package from source distribution
           # ARCHITECTURE environment variable controls target platform tagging
           - script: |

eng/pipelines/build-whl-pipeline.yml

@@ -168,6 +168,13 @@ jobs:
         TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs'
       displayName: 'Place PDB file into artifacts directory'
 
+    # Extract mssql_py_core from NuGet into repo root before building the wheel
+    - task: PowerShell@2
+      displayName: 'Install mssql_py_core from NuGet'
+      inputs:
+        targetType: 'filePath'
+        filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1'
+
     # Build wheel package for the current architecture
     - script: |
         python -m pip install --upgrade pip
@@ -182,7 +189,7 @@ jobs:
         SourceFolder: '$(Build.SourcesDirectory)\dist'
         Contents: '*.whl'
         TargetFolder: '$(Build.ArtifactStagingDirectory)\dist'
-      displayName: 'Collect wheel package'    
+      displayName: 'Collect wheel package'
 
     # Publish the collected .pyd file(s) as build artifacts
     - task: PublishBuildArtifacts@1
@@ -342,6 +349,11 @@ jobs:
       env:
         DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
 
+    # Extract mssql_py_core from NuGet into repo root before building the wheel
+    - script: |
+        bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh
+      displayName: 'Install mssql_py_core from NuGet'
+
     # Build wheel package for universal2
     - script: |
         python -m pip install --upgrade pip
@@ -356,7 +368,7 @@ jobs:
         Contents: '*.whl'
         TargetFolder: '$(Build.ArtifactStagingDirectory)/dist'
       displayName: 'Collect wheel package'
-    
+
     # Publish the collected .so file(s) as build artifacts
     - task: PublishBuildArtifacts@1
       condition: succeededOrFailed()
@@ -444,10 +456,10 @@ jobs:
             if command -v dnf >/dev/null 2>&1; then
               dnf -y update || true
               # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache
-              dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true
+              dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true
             elif command -v yum >/dev/null 2>&1; then
               yum -y update || true
-              yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true
+              yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true
             else
               echo "No dnf/yum found in manylinux image" >&2
             fi
@@ -463,7 +475,7 @@ jobs:
             set -euxo pipefail
             apk update || true
             # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache
-            apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true
+            apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true
 
             # Quick visibility for logs
             echo "---- tool versions ----"
@@ -490,12 +502,16 @@ jobs:
               PY=/opt/python/${PYBIN}-${PYBIN}/bin/python;
               test -x \$PY || { echo 'Python \$PY missing'; exit 0; }  # skip if not present
               ln -sf \$PY /usr/local/bin/python;
+              ln -sf \$PY /usr/local/bin/python3;
               python -m pip install -U pip setuptools wheel pybind11;
               echo 'python:' \$(python -V); which python;
               # 👉 run from the directory that has CMakeLists.txt
               cd /workspace/mssql_python/pybind;
               bash build.sh;
 
+              # Extract mssql_py_core from NuGet for this Python version
+              bash /workspace/eng/scripts/install-mssql-py-core.sh;
+
               # back to repo root to build the wheel
               cd /workspace;
               python setup.py bdist_wheel;
@@ -510,12 +526,16 @@ jobs:
               PY=/opt/python/${PYBIN}-${PYBIN}/bin/python;
               test -x \$PY || { echo 'Python \$PY missing'; exit 0; }  # skip if not present
               ln -sf \$PY /usr/local/bin/python;
+              ln -sf \$PY /usr/local/bin/python3;
               python -m pip install -U pip setuptools wheel pybind11;
               echo 'python:' \$(python -V); which python;
               # 👉 run from the directory that has CMakeLists.txt
               cd /workspace/mssql_python/pybind;
               bash build.sh;
 
+              # Extract mssql_py_core from NuGet for this Python version
+              bash /workspace/eng/scripts/install-mssql-py-core.sh;
+
               # back to repo root to build the wheel
               cd /workspace;
               python setup.py bdist_wheel;

eng/pipelines/pr-validation-pipeline.yml

@@ -230,6 +230,13 @@ jobs:
       build.bat x64
     displayName: 'Build .pyd file'
 
+  # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests)
+  - task: PowerShell@2
+    displayName: 'Install mssql_py_core from NuGet wheels'
+    inputs:
+      targetType: 'filePath'
+      filePath: 'eng/scripts/install-mssql-py-core.ps1'
+
   # Run tests for LocalDB
   - script: |
       python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear
@@ -497,6 +504,12 @@ jobs:
       ./build.sh
     displayName: 'Build pybind bindings (.so)'
 
+  # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests)
+  - script: |
+      chmod +x eng/scripts/install-mssql-py-core.sh
+      ./eng/scripts/install-mssql-py-core.sh
+    displayName: 'Install mssql_py_core from NuGet wheels'
+
   - script: |
       echo "Build successful, running tests now"
       python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear
@@ -669,6 +682,15 @@ jobs:
       "
     displayName: 'Build pybind bindings (.so) in $(distroName) container'
 
+  # Install mssql_py_core from NuGet wheel package inside container
+  - script: |
+      docker exec test-container-$(distroName) bash -c "
+        source /opt/venv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in $(distroName) container'
+
   - script: |
       # Uninstall ODBC Driver before running tests
       docker exec test-container-$(distroName) bash -c "
@@ -984,6 +1006,15 @@ jobs:
     displayName: 'Build pybind bindings (.so) in $(distroName) ARM64 container'
     retryCountOnTaskFailure: 2
 
+  # Install mssql_py_core from NuGet wheel package inside ARM64 container
+  - script: |
+      docker exec test-container-$(distroName)-$(archName) bash -c "
+        source /opt/venv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in $(distroName) ARM64 container'
+
   - script: |
       # Uninstall ODBC Driver before running tests
       docker exec test-container-$(distroName)-$(archName) bash -c "
@@ -1192,6 +1223,15 @@ jobs:
       "
     displayName: 'Build pybind bindings (.so) in RHEL 9 container'
 
+  # Install mssql_py_core from NuGet wheel package inside RHEL 9 container
+  - script: |
+      docker exec test-container-rhel9 bash -c "
+        source myvenv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in RHEL 9 container'
+
   - script: |
       # Uninstall ODBC Driver before running tests
       docker exec test-container-rhel9 bash -c "
@@ -1411,6 +1451,15 @@ jobs:
     displayName: 'Build pybind bindings (.so) in RHEL 9 ARM64 container'
     retryCountOnTaskFailure: 2
 
+  # Install mssql_py_core from NuGet wheel package inside RHEL 9 ARM64 container
+  - script: |
+      docker exec test-container-rhel9-arm64 bash -c "
+        source myvenv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in RHEL 9 ARM64 container'
+
   - script: |
       # Uninstall ODBC Driver before running tests
       docker exec test-container-rhel9-arm64 bash -c "
@@ -1638,6 +1687,15 @@ jobs:
       "
     displayName: 'Build pybind bindings (.so) in Alpine x86_64 container'
 
+  # Install mssql_py_core from NuGet wheel package inside Alpine container
+  - script: |
+      docker exec test-container-alpine bash -c "
+        source /workspace/venv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in Alpine x86_64 container'
+
   - script: |
       # Uninstall ODBC Driver before running tests to use bundled libraries
       docker exec test-container-alpine bash -c "
@@ -1883,6 +1941,15 @@ jobs:
     displayName: 'Build pybind bindings (.so) in Alpine ARM64 container'
     retryCountOnTaskFailure: 2
 
+  # Install mssql_py_core from NuGet wheel package inside Alpine ARM64 container
+  - script: |
+      docker exec test-container-alpine-arm64 bash -c "
+        source /workspace/venv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in Alpine ARM64 container'
+
   - script: |
       # Uninstall ODBC Driver before running tests to use bundled libraries
       docker exec test-container-alpine-arm64 bash -c "
@@ -2005,6 +2072,13 @@ jobs:
       build.bat x64
     displayName: 'Build .pyd file'
 
+  # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests)
+  - task: PowerShell@2
+    displayName: 'Install mssql_py_core from NuGet wheels'
+    inputs:
+      targetType: 'filePath'
+      filePath: 'eng/scripts/install-mssql-py-core.ps1'
+
   - script: |
       python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear
     displayName: 'Run tests on Azure SQL Database'
@@ -2047,6 +2121,12 @@ jobs:
       ./build.sh
     displayName: 'Build pybind bindings (.so)'
 
+  # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests)
+  - script: |
+      chmod +x eng/scripts/install-mssql-py-core.sh
+      ./eng/scripts/install-mssql-py-core.sh
+    displayName: 'Install mssql_py_core from NuGet wheels'
+
   - script: |
       python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear
     displayName: 'Run tests on Azure SQL Database'
@@ -2118,6 +2198,15 @@ jobs:
       "
     displayName: 'Build pybind bindings (.so) in Ubuntu container'
 
+  # Install mssql_py_core from NuGet wheel package inside Ubuntu container
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        source /opt/venv/bin/activate
+        chmod +x eng/scripts/install-mssql-py-core.sh
+        ./eng/scripts/install-mssql-py-core.sh
+      "
+    displayName: 'Install mssql_py_core in Ubuntu container'
+
   - script: |
       docker exec test-container-ubuntu-azuresql bash -c "
         export DEBIAN_FRONTEND=noninteractive