Enhance build workflow to validate interpreter and wheel mode compatibility; update README for clarity on free-threaded builds and limited API usage

kosiew · kosiew · commit 0ebd47623ede · 2026-02-11T13:53:19.000+08:00
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -143,10 +143,45 @@ jobs:
         with:
             enable-cache: true
 
+      - name: Validate interpreter/wheel mode
+        shell: bash
+        env:
+          WHEEL_MODE: ${{ endsWith(matrix.python-version, 't') && 'freethreaded' || 'abi3' }}
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
+        run: |
+          python - <<'PY'
+          import os
+          import sys
+          import sysconfig
+
+          mode = os.environ["WHEEL_MODE"]
+          features = os.environ["MATURIN_FEATURES"]
+          is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
+
+          print(sys.version)
+          print(f"Py_GIL_DISABLED={int(is_free_threaded)}")
+          print(f"wheel_mode={mode}")
+          print(f"maturin_features={features}")
+
+          if mode == "abi3" and is_free_threaded:
+              raise SystemExit("Invalid matrix: free-threaded interpreter cannot build abi3 wheels")
+          if mode == "freethreaded" and not is_free_threaded:
+              raise SystemExit("Invalid matrix: freethreaded wheel mode requires a free-threaded interpreter")
+          PY
+
+      - name: Show resolved PyO3 versions
+        shell: bash
+        run: |
+          cargo tree -e no-dev -i pyo3
+          cargo tree -e no-dev -i pyo3-build-config
+
       - name: Build Python package
+        shell: bash
+        env:
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
         run: |
           uv sync --dev --no-install-package datafusion
-          uv run --no-project maturin build --release --strip --features substrait --interpreter python
+          uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter python
 
       - name: List Windows wheels
         if: matrix.os == 'windows-latest'
@@ -213,10 +248,45 @@ jobs:
         with:
             enable-cache: true
 
+      - name: Validate interpreter/wheel mode
+        shell: bash
+        env:
+          WHEEL_MODE: ${{ endsWith(matrix.python-version, 't') && 'freethreaded' || 'abi3' }}
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
+        run: |
+          python - <<'PY'
+          import os
+          import sys
+          import sysconfig
+
+          mode = os.environ["WHEEL_MODE"]
+          features = os.environ["MATURIN_FEATURES"]
+          is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
+
+          print(sys.version)
+          print(f"Py_GIL_DISABLED={int(is_free_threaded)}")
+          print(f"wheel_mode={mode}")
+          print(f"maturin_features={features}")
+
+          if mode == "abi3" and is_free_threaded:
+              raise SystemExit("Invalid matrix: free-threaded interpreter cannot build abi3 wheels")
+          if mode == "freethreaded" and not is_free_threaded:
+              raise SystemExit("Invalid matrix: freethreaded wheel mode requires a free-threaded interpreter")
+          PY
+
+      - name: Show resolved PyO3 versions
+        shell: bash
+        run: |
+          cargo tree -e no-dev -i pyo3
+          cargo tree -e no-dev -i pyo3-build-config
+
       - name: Build Python package
+        shell: bash
+        env:
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
         run: |
           uv sync --dev --no-install-package datafusion
-          uv run --no-project maturin build --release --strip --features substrait --interpreter python
+          uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter python
 
       - name: List Mac wheels
         run: find target/wheels/
@@ -237,8 +307,12 @@ jobs:
         include:
           - interpreter: "3.10 3.11 3.12 3.13 3.14"
             artifact-label: cp310-314
+            wheel-mode: abi3
+            maturin-features: protoc,substrait,py-limited-api
           - interpreter: "3.13t 3.14t"
             artifact-label: cp313t-314t
+            wheel-mode: freethreaded
+            maturin-features: protoc,substrait
     steps:
       - uses: actions/checkout@v5
       - run: rm LICENSE.txt
@@ -248,6 +322,20 @@ jobs:
           name: python-wheel-license
           path: .
       - run: cat LICENSE.txt
+      - name: Validate wheel mode matrix
+        shell: bash
+        run: |
+          echo "interpreter=${{ matrix.interpreter }}"
+          echo "wheel_mode=${{ matrix.wheel-mode }}"
+          echo "maturin_features=${{ matrix.maturin-features }}"
+          if [[ "${{ matrix.wheel-mode }}" == "abi3" && "${{ matrix.interpreter }}" == *t* ]]; then
+            echo "Invalid matrix: abi3 mode cannot use free-threaded interpreters"
+            exit 1
+          fi
+          if [[ "${{ matrix.wheel-mode }}" == "freethreaded" && "${{ matrix.interpreter }}" != *t* ]]; then
+            echo "Invalid matrix: freethreaded mode requires free-threaded interpreters"
+            exit 1
+          fi
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
@@ -258,7 +346,7 @@ jobs:
           manylinux: auto
           rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
           interpreter: ${{ matrix.interpreter }}
-          args: --release --manylinux 2014 --features protoc,substrait
+          args: --release --manylinux 2014 --features ${{ matrix.maturin-features }}
       - name: Archive wheels
         uses: actions/upload-artifact@v4
         with:
@@ -275,8 +363,12 @@ jobs:
         include:
           - interpreter: "3.10 3.11 3.12 3.13 3.14"
             artifact-label: cp310-314
+            wheel-mode: abi3
+            maturin-features: protoc,substrait,py-limited-api
           - interpreter: "3.13t 3.14t"
             artifact-label: cp313t-314t
+            wheel-mode: freethreaded
+            maturin-features: protoc,substrait
     steps:
       - uses: actions/checkout@v5
       - run: rm LICENSE.txt
@@ -286,6 +378,20 @@ jobs:
           name: python-wheel-license
           path: .
       - run: cat LICENSE.txt
+      - name: Validate wheel mode matrix
+        shell: bash
+        run: |
+          echo "interpreter=${{ matrix.interpreter }}"
+          echo "wheel_mode=${{ matrix.wheel-mode }}"
+          echo "maturin_features=${{ matrix.maturin-features }}"
+          if [[ "${{ matrix.wheel-mode }}" == "abi3" && "${{ matrix.interpreter }}" == *t* ]]; then
+            echo "Invalid matrix: abi3 mode cannot use free-threaded interpreters"
+            exit 1
+          fi
+          if [[ "${{ matrix.wheel-mode }}" == "freethreaded" && "${{ matrix.interpreter }}" != *t* ]]; then
+            echo "Invalid matrix: freethreaded mode requires free-threaded interpreters"
+            exit 1
+          fi
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
@@ -297,7 +403,7 @@ jobs:
           manylinux: 2_28
           rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
           interpreter: ${{ matrix.interpreter }}
-          args: --release --features protoc,substrait
+          args: --release --features ${{ matrix.maturin-features }}
       - name: Archive wheels
         uses: actions/upload-artifact@v4
         with:
diff --git a/Cargo.toml b/Cargo.toml
@@ -40,6 +40,7 @@ include = [
 default = ["mimalloc"]
 protoc = ["datafusion-substrait/protoc"]
 substrait = ["dep:datafusion-substrait"]
+py-limited-api = ["pyo3/abi3-py310"]
 
 [dependencies]
 tokio = { version = "1.47", features = [
@@ -50,7 +51,6 @@ tokio = { version = "1.47", features = [
 ] }
 pyo3 = { version = "0.26", features = [
     "extension-module",
-    "abi3-py310",
 ] }
 pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
 pyo3-log = "0.13.2"
diff --git a/dev/release/README.md b/dev/release/README.md
@@ -130,9 +130,9 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
 datafusion-22.0.0-cp37-abi3-win_amd64.whl
 ```
 
-Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and ignore the `abi3` setting. We keep
-the `abi3-py310` feature enabled to preserve the stable-ABI wheels for GIL-enabled Python, and we additionally ship
-version-specific `cp313t`/`cp314t` wheels for the free-threaded builds.
+Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and cannot use the limited API (`abi3`).
+The release workflow enables Cargo feature `py-limited-api` for GIL-enabled wheels and disables it for free-threaded
+builds, producing version-specific `cp313t`/`cp314t` wheels.
 
 Upload the wheels to testpypi.
 
