Skip to content

Commit 5f38b20

Browse files
timsaucertimsaucer
committed
Initial commit for free threaded python support
1 parent afaeccb commit 5f38b20

5 files changed

Lines changed: 196 additions & 82 deletions

File tree

‎.github/actions/build-wheel/action.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Composite action that builds a datafusion-python wheel with maturin.
19+
# Centralises the abi3-vs-free-threaded argument logic so platform jobs
20+
# stay short and changes to wheel-build flags happen in one place.
21+
22+
name: "Build wheel"
23+
description: "Build datafusion-python wheel with maturin (abi3 or free-threaded)"
24+
25+
inputs:
26+
target:
27+
description: "Rust target triple (e.g. x86_64-unknown-linux-gnu). Required when manylinux is set; ignored for native builds."
28+
required: false
29+
default: ""
30+
python-tag:
31+
description: "abi3 (covers 3.10..3.14 GIL builds) or a free-threaded interpreter such as 3.13t / 3.14t"
32+
required: true
33+
build-mode:
34+
description: "release or debug"
35+
required: true
36+
features:
37+
description: "Comma-separated extra features (in addition to those implied by the python-tag)"
38+
required: false
39+
default: "substrait"
40+
manylinux:
41+
description: "manylinux tag for maturin-action (e.g. 2_28). Leave empty to use uv-run maturin natively."
42+
required: false
43+
default: ""
44+
out-dir:
45+
description: "Output directory for built wheels"
46+
required: false
47+
default: "dist"
48+
49+
outputs:
50+
args:
51+
description: "Computed maturin args (for debugging)"
52+
value: ${{ steps.args.outputs.args }}
53+
54+
runs:
55+
using: "composite"
56+
steps:
57+
- name: Compute maturin args
58+
id: args
59+
shell: bash
60+
run: |
61+
set -euo pipefail
62+
FEATURES="${{ inputs.features }}"
63+
TAG="${{ inputs.python-tag }}"
64+
if [ "$TAG" = "abi3" ]; then
65+
# Default features include the `abi3` cargo feature.
66+
# One wheel covers Python 3.10..3.14 (GIL builds only).
67+
BUILD_ARGS="--features ${FEATURES}"
68+
else
69+
# Free-threaded build: disable abi3, force mimalloc back in, pin interpreter.
70+
BUILD_ARGS="--no-default-features --features mimalloc,${FEATURES} --interpreter python${TAG}"
71+
fi
72+
if [ "${{ inputs.build-mode }}" = "release" ]; then
73+
BUILD_ARGS="--release --strip ${BUILD_ARGS}"
74+
fi
75+
BUILD_ARGS="${BUILD_ARGS} --out ${{ inputs.out-dir }}"
76+
echo "args=${BUILD_ARGS}" >> "$GITHUB_OUTPUT"
77+
echo "maturin args: ${BUILD_ARGS}"
78+
79+
- name: Build via maturin-action (manylinux container)
80+
if: inputs.manylinux != ''
81+
uses: PyO3/maturin-action@v1
82+
with:
83+
target: ${{ inputs.target }}
84+
manylinux: ${{ inputs.manylinux }}
85+
maturin-version: "1.8.1"
86+
args: ${{ steps.args.outputs.args }}
87+
rustup-components: rust-std
88+
89+
- name: Build via native maturin
90+
if: inputs.manylinux == ''
91+
shell: bash
92+
run: uv run --no-project maturin build ${{ steps.args.outputs.args }}

‎.github/workflows/build.yml‎

Lines changed: 71 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,12 @@ jobs:
135135
# ============================================
136136
build-manylinux-x86_64:
137137
needs: [generate-license, lint-rust, lint-python]
138-
name: ManyLinux x86_64
138+
name: ManyLinux x86_64 (${{ matrix.python-tag }})
139139
runs-on: ubuntu-latest
140+
strategy:
141+
fail-fast: false
142+
matrix:
143+
python-tag: ["abi3", "3.13t", "3.14t"]
140144
steps:
141145
- uses: actions/checkout@v6
142146

@@ -153,7 +157,7 @@ jobs:
153157
- name: Cache Cargo
154158
uses: Swatinem/rust-cache@v2
155159
with:
156-
key: ${{ inputs.build_mode }}
160+
key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}
157161

158162
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
159163
with:
@@ -172,25 +176,18 @@ jobs:
172176
free -h
173177
swapon --show
174178
175-
- name: Build (release mode)
176-
uses: PyO3/maturin-action@v1
177-
if: inputs.build_mode == 'release'
178-
with:
179-
target: x86_64-unknown-linux-gnu
180-
manylinux: "2_28"
181-
args: --release --strip --features protoc,substrait --out dist
182-
rustup-components: rust-std
183-
184-
- name: Build (debug mode)
185-
uses: PyO3/maturin-action@v1
186-
if: inputs.build_mode == 'debug'
179+
- name: Build wheel
180+
uses: ./.github/actions/build-wheel
187181
with:
188182
target: x86_64-unknown-linux-gnu
183+
python-tag: ${{ matrix.python-tag }}
184+
build-mode: ${{ inputs.build_mode }}
185+
features: "protoc,substrait"
189186
manylinux: "2_28"
190-
args: --features protoc,substrait --out dist
191-
rustup-components: rust-std
192187

188+
# FFI test wheel only needs to be built once per platform; gate to abi3.
193189
- name: Build FFI test library
190+
if: matrix.python-tag == 'abi3'
194191
uses: PyO3/maturin-action@v1
195192
with:
196193
target: x86_64-unknown-linux-gnu
@@ -202,10 +199,11 @@ jobs:
202199
- name: Archive wheels
203200
uses: actions/upload-artifact@v7
204201
with:
205-
name: dist-manylinux-x86_64
202+
name: dist-manylinux-x86_64-${{ matrix.python-tag }}
206203
path: dist/*
207204

208205
- name: Archive FFI test wheel
206+
if: matrix.python-tag == 'abi3'
209207
uses: actions/upload-artifact@v7
210208
with:
211209
name: test-ffi-manylinux-x86_64
@@ -216,8 +214,12 @@ jobs:
216214
# ============================================
217215
build-manylinux-aarch64:
218216
needs: [generate-license, lint-rust, lint-python]
219-
name: ManyLinux arm64
217+
name: ManyLinux arm64 (${{ matrix.python-tag }})
220218
runs-on: ubuntu-24.04-arm
219+
strategy:
220+
fail-fast: false
221+
matrix:
222+
python-tag: ["abi3", "3.13t", "3.14t"]
221223
steps:
222224
- uses: actions/checkout@v6
223225

@@ -234,7 +236,7 @@ jobs:
234236
- name: Cache Cargo
235237
uses: Swatinem/rust-cache@v2
236238
with:
237-
key: ${{ inputs.build_mode }}
239+
key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}
238240

239241
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
240242
with:
@@ -253,43 +255,34 @@ jobs:
253255
free -h
254256
swapon --show
255257
256-
- name: Build (release mode)
257-
uses: PyO3/maturin-action@v1
258-
if: inputs.build_mode == 'release'
259-
with:
260-
target: aarch64-unknown-linux-gnu
261-
manylinux: "2_28"
262-
args: --release --strip --features protoc,substrait --out dist
263-
rustup-components: rust-std
264-
265-
- name: Build (debug mode)
266-
uses: PyO3/maturin-action@v1
267-
if: inputs.build_mode == 'debug'
258+
- name: Build wheel
259+
uses: ./.github/actions/build-wheel
268260
with:
269261
target: aarch64-unknown-linux-gnu
262+
python-tag: ${{ matrix.python-tag }}
263+
build-mode: ${{ inputs.build_mode }}
264+
features: "protoc,substrait"
270265
manylinux: "2_28"
271-
args: --features protoc,substrait --out dist
272-
rustup-components: rust-std
273266

274267
- name: Archive wheels
275268
uses: actions/upload-artifact@v7
276269
if: inputs.build_mode == 'release'
277270
with:
278-
name: dist-manylinux-aarch64
271+
name: dist-manylinux-aarch64-${{ matrix.python-tag }}
279272
path: dist/*
280273

281274
# ============================================
282275
# Build - macOS arm64 / Windows
283276
# ============================================
284277
build-python-mac-win:
285278
needs: [generate-license, lint-rust, lint-python]
286-
name: macOS arm64 & Windows
279+
name: macOS arm64 & Windows (${{ matrix.python-tag }} / ${{ matrix.os }})
287280
runs-on: ${{ matrix.os }}
288281
strategy:
289282
fail-fast: false
290283
matrix:
291-
python-version: ["3.10"]
292284
os: [macos-latest, windows-latest]
285+
python-tag: ["abi3", "3.13t", "3.14t"]
293286
steps:
294287
- uses: actions/checkout@v6
295288

@@ -305,7 +298,14 @@ jobs:
305298
- name: Cache Cargo
306299
uses: Swatinem/rust-cache@v2
307300
with:
308-
key: ${{ inputs.build_mode }}
301+
key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}
302+
303+
- name: Setup Python (free-threaded)
304+
if: matrix.python-tag != 'abi3'
305+
uses: actions/setup-python@v6
306+
with:
307+
python-version: ${{ matrix.python-tag }}
308+
freethreaded: true
309309

310310
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
311311
with:
@@ -318,22 +318,22 @@ jobs:
318318
repo-token: ${{ secrets.GITHUB_TOKEN }}
319319

320320
- name: Install dependencies
321+
if: matrix.python-tag == 'abi3'
321322
run: uv sync --dev --no-install-package datafusion
322323

323-
# Run clippy BEFORE maturin so we can avoid rebuilding. The features must match
324-
# exactly the features used by maturin. Linux maturin builds need to happen in a
325-
# container so only run this for our mac runner.
324+
# Clippy is interpreter-agnostic; run once per OS (against the abi3 entry)
325+
# so the matrix doesn't pay the cost three times.
326326
- name: Run Clippy
327-
if: matrix.os != 'windows-latest'
327+
if: matrix.os != 'windows-latest' && matrix.python-tag == 'abi3'
328328
run: cargo clippy --no-deps --all-targets --features substrait -- -D warnings
329329

330-
- name: Build Python package (release mode)
331-
if: inputs.build_mode == 'release'
332-
run: uv run --no-project maturin build --release --strip --features substrait
333-
334-
- name: Build Python package (debug mode)
335-
if: inputs.build_mode != 'release'
336-
run: uv run --no-project maturin build --features substrait
330+
- name: Build wheel
331+
uses: ./.github/actions/build-wheel
332+
with:
333+
python-tag: ${{ matrix.python-tag }}
334+
build-mode: ${{ inputs.build_mode }}
335+
features: "substrait"
336+
out-dir: "target/wheels"
337337

338338
- name: List Windows wheels
339339
if: matrix.os == 'windows-latest'
@@ -350,7 +350,7 @@ jobs:
350350
uses: actions/upload-artifact@v7
351351
if: inputs.build_mode == 'release'
352352
with:
353-
name: dist-${{ matrix.os }}
353+
name: dist-${{ matrix.os }}-${{ matrix.python-tag }}
354354
path: target/wheels/*
355355

356356
# ============================================
@@ -359,11 +359,12 @@ jobs:
359359
build-macos-x86_64:
360360
if: inputs.build_mode == 'release'
361361
needs: [generate-license, lint-rust, lint-python]
362+
name: macOS x86_64 (${{ matrix.python-tag }})
362363
runs-on: macos-15-intel
363364
strategy:
364365
fail-fast: false
365366
matrix:
366-
python-version: ["3.10"]
367+
python-tag: ["abi3", "3.13t", "3.14t"]
367368
steps:
368369
- uses: actions/checkout@v6
369370

@@ -379,7 +380,14 @@ jobs:
379380
- name: Cache Cargo
380381
uses: Swatinem/rust-cache@v2
381382
with:
382-
key: ${{ inputs.build_mode }}
383+
key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}
384+
385+
- name: Setup Python (free-threaded)
386+
if: matrix.python-tag != 'abi3'
387+
uses: actions/setup-python@v6
388+
with:
389+
python-version: ${{ matrix.python-tag }}
390+
freethreaded: true
383391

384392
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
385393
with:
@@ -392,19 +400,24 @@ jobs:
392400
repo-token: ${{ secrets.GITHUB_TOKEN }}
393401

394402
- name: Install dependencies
403+
if: matrix.python-tag == 'abi3'
395404
run: uv sync --dev --no-install-package datafusion
396405

397-
- name: Build (release mode)
398-
run: |
399-
uv run --no-project maturin build --release --strip --features substrait
406+
- name: Build wheel
407+
uses: ./.github/actions/build-wheel
408+
with:
409+
python-tag: ${{ matrix.python-tag }}
410+
build-mode: ${{ inputs.build_mode }}
411+
features: "substrait"
412+
out-dir: "target/wheels"
400413

401414
- name: List Mac wheels
402415
run: find target/wheels/
403416

404417
- name: Archive wheels
405418
uses: actions/upload-artifact@v7
406419
with:
407-
name: dist-macos-aarch64
420+
name: dist-macos-aarch64-${{ matrix.python-tag }}
408421
path: target/wheels/*
409422

410423
# ============================================
@@ -509,11 +522,12 @@ jobs:
509522
with:
510523
enable-cache: true
511524

512-
# Download the Linux wheel built in the previous job
525+
# Download the Linux wheel built in the previous job.
526+
# Docs only need the abi3 wheel — interpreter doesn't matter for sphinx.
513527
- name: Download pre-built Linux wheel
514528
uses: actions/download-artifact@v8
515529
with:
516-
name: dist-manylinux-x86_64
530+
name: dist-manylinux-x86_64-abi3
517531
path: wheels/
518532

519533
# Install from the pre-built wheels

0 commit comments

Comments
 (0)