Skip to content

GH-36411: [C++][Python] Use meson-python for PyArrow build system #45854

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WillAyd wants to merge 3 commits into
base: main
Choose a base branch
from
+993 −1,550
Open

GH-36411: [C++][Python] Use meson-python for PyArrow build system #45854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4256c93
GH-36411: [C++][Python] Use meson-python for PyArrow build system
WillAyd Mar 18, 2025
bf88854
Hack for grpc example
WillAyd Dec 20, 2025
ce85d06
Force minimum nodejs version for azurite to work
raulcd Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ jobs:
shell: bash
run: |
gem install test-unit openssl
pip install "cython>=3.1" setuptools pytest requests setuptools-scm
pip install "cython>=3.1" meson-python pytest requests setuptools-scm
- name: Run Release Test
shell: bash
run: |
Expand Down
26 changes: 23 additions & 3 deletions .github/workflows/python.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ jobs:
with:
python-version: 3.12
- name: Setup Archery
run: pip install -e dev/archery[docker]
run: pip install -e dev/archery[docker] meson-python
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
Expand Down Expand Up @@ -175,7 +175,7 @@ jobs:
ARROW_BUILD_TESTS: OFF
PYARROW_TEST_LARGE_MEMORY: ON
# Current oldest supported version according to https://endoflife.date/macos
MACOSX_DEPLOYMENT_TARGET: 12.0
MACOSX_DEPLOYMENT_TARGET: "12.0"
Copy link
Contributor Author

@WillAyd WillAyd Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I enclosed this in parentheses so that it gets evaluated as the string "12.0" and not the number; otherwise some functionality in Meson was failing to unpack the major/minor version

Somewhat tangentially it looks like the minimum supported version right now is 14.0 - worth updating?

steps:
- name: Checkout Arrow
uses: actions/checkout@v6
Expand Down Expand Up @@ -288,11 +288,31 @@ jobs:
env:
# We can invalidate the current cache by updating this.
CACHE_VERSION: "2025-09-16.1"
- name: Install conda
shell: cmd
run: |
curl -L https://github.com/conda-forge/miniforge/releases/download/25.11.0-0/Miniforge3-25.11.0-0-Windows-x86_64.exe -o miniforge.exe
start /wait "" .\miniforge.exe /S /D=C:\miniforge
del .\miniforge.exe
C:\miniforge\Scripts\conda.exe init cmd.exe
echo C:\miniforge\Scripts;C:\miniforge\Library\bin;C:\miniforge\condabin>> %GITHUB_PATH%
- name: Create conda environment
shell: cmd
run: |
conda env create -n pyarrow-dev -f .\ci\conda_env_cpp.txt -y
conda env update -n pyarrow-dev -f .\ci\conda_env_python.txt
- name: Build Arrow C++ and PyArrow
shell: cmd
env:
PYTHON_CMD: "python"
run: |
call "ci\scripts\python_build.bat" %cd% "${{ steps.path-info.outputs.usr-windows-dir }}"
call activate pyarrow-dev || exit /B 1
conda list
call "ci\scripts\python_build.bat" %cd% "%CONDA_PREFIX%"
- name: Test PyArrow
shell: cmd
env:
PYTHON_CMD: "python"
run: |
call activate pyarrow-dev
call "ci\scripts\python_test.bat" %cd%
4 changes: 2 additions & 2 deletions ci/conda_env_cpp.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ libboost-devel
libgrpc
libprotobuf
libutf8proc
lz4-c
lz4-c>=1.10.0
make
meson
ninja
nodejs
nodejs>=16
orc<2.1.0
pkg-config
python
Expand Down
2 changes: 1 addition & 1 deletion ci/conda_env_python.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ cython>=3.1
cloudpickle
fsspec
hypothesis
meson-python
numpy>=1.16.6
pytest
pytest-faulthandler
s3fs>=2023.10.0
setuptools>=77
setuptools_scm>=8
1 change: 1 addition & 0 deletions ci/conda_env_sphinx.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ cython>3.1.1
doxygen
ipython
linkify-it-py
meson-python
# We can't install linuxdoc by conda. We install linuxdoc by pip in
# ci/dockerfiles/conda-python-pandas.dockerfile.
# linuxdoc
Expand Down
114 changes: 97 additions & 17 deletions ci/scripts/python_build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,30 +110,110 @@ ccache -sv

echo "=== Building Python ==="
set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
set PYARROW_BUILD_VERBOSE=1
set PYARROW_BUNDLE_ARROW_CPP=ON
set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
set PYARROW_WITH_ACERO=%ARROW_ACERO%
set PYARROW_WITH_DATASET=%ARROW_DATASET%
set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
set PYARROW_WITH_GCS=%ARROW_GCS%
set PYARROW_WITH_HDFS=%ARROW_HDFS%
set PYARROW_WITH_ORC=%ARROW_ORC%
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
set PYARROW_WITH_S3=%ARROW_S3%
set ARROW_HOME=%CMAKE_INSTALL_PREFIX%
set CMAKE_PREFIX_PATH=%CMAKE_INSTALL_PREFIX%
if %ARROW_ACERO% == ON (
set PYARROW_WITH_ACERO=enabled
) else if %ARROW_ACERO% == OFF (
set PYARROW_WITH_ACERO=disabled
) else (
set PYARROW_WITH_ACERO=auto
)
if %ARROW_DATASET% == ON (
set PYARROW_WITH_DATASET=enabled
) else if %ARROW_DATASET% == OFF (
set PYARROW_WITH_DATASET=disabled
) else (
set PYARROW_WITH_DATASET=auto
)
if %ARROW_FLIGHT% == ON (
set PYARROW_WITH_FLIGHT=enabled
) else if %ARROW_FLIGHT% == OFF (
set PYARROW_WITH_FLIGHT=disabled
) else (
set PYARROW_WITH_FLIGHT=auto
)
if %ARROW_GANDIVA% == ON (
set PYARROW_WITH_GANDIVA=enabled
) else if %ARROW_GANDIVA% == OFF (
set PYARROW_WITH_GANDIVA=disabled
) else (
set PYARROW_WITH_GANDIVA=auto
)
if %ARROW_GCS% == ON (
set PYARROW_WITH_GCS=enabled
) else if %ARROW_GCS% == OFF (
set PYARROW_WITH_GCS=disabled
) else (
set PYARROW_WITH_GCS=auto
)
if %ARROW_HDFS% == ON (
set PYARROW_WITH_HDFS=enabled
) else if %ARROW_HDFS% == OFF (
set PYARROW_WITH_HDFS=disabled
) else (
set PYARROW_WITH_HDFS=auto
)
if %ARROW_ORC% == ON (
set PYARROW_WITH_ORC=enabled
) else if %ARROW_ORC% == OFF (
set PYARROW_WITH_ORC=disabled
) else (
set PYARROW_WITH_ORC=auto
)
if %ARROW_PARQUET% == ON (
set PYARROW_WITH_PARQUET=enabled
) else if %ARROW_PARQUET% == OFF (
set PYARROW_WITH_PARQUET=disabled
) else (
set PYARROW_WITH_PARQUET=auto
)
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
) else if %ARROW_ACERO% == OFF (
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
) else (
set PYARROW_WITH_PARQUET_ENCRYPTION=auto
)
if %ARROW_SUBSTRAIT% == ON (
set PYARROW_WITH_SUBSTRAIT=enabled
) else if %ARROW_SUBSTRAIT% == OFF (
set PYARROW_WITH_SUBSTRAIT=disabled
) else (
set PYARROW_WITH_SUBSTRAIT=auto
)
if %ARROW_S3% == ON (
set PYARROW_WITH_S3=enabled
) else if %ARROW_S3% == OFF (
set PYARROW_WITH_S3=disabled
) else (
set PYARROW_WITH_S3=auto
)
if %CMAKE_BUILD_TYPE% == Release (
set MESON_BUILD_TYPE=release
) else (
set MESON_BUILD_TYPE=debug
)

pushd %SOURCE_DIR%\python

@REM Install Python build dependencies
%PYTHON_CMD% -m pip install --upgrade pip || exit /B 1
%PYTHON_CMD% -m pip install -r requirements-build.txt || exit /B 1
%PYTHON_CMD% -m pip install build || exit /B 1

@REM Build PyArrow
%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv . || exit /B 1
%PYTHON_CMD% -m build --wheel --no-isolation . ^
-Csetup-args="-Dbuildtype=%MESON_BUILD_TYPE%" ^
-Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^
-Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^
-Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^
-Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^
-Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^
-Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^
-Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^
-Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^
-Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^
-Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^
-Csetup-args="-Ds3=%PYARROW_WITH_S3%" || exit /B 1
%PYTHON_CMD% -m pip install --no-index --find-links .\dist\ pyarrow

popd
133 changes: 99 additions & 34 deletions ci/scripts/python_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,44 +39,76 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

case "$(uname)" in
Linux)
n_jobs=$(nproc)
;;
Darwin)
n_jobs=$(sysctl -n hw.ncpu)
;;
MINGW*)
n_jobs=${NUMBER_OF_PROCESSORS:-1}
;;
*)
n_jobs=${NPROC:-1}
;;
esac

if [ -n "${CONDA_PREFIX}" ]; then
echo -e "===\n=== Conda environment for build\n==="
conda list
fi

export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}

export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}

export PYARROW_PARALLEL=${n_jobs}
PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_S3=$(case "$ARROW_S3" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)

: "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
export CMAKE_PREFIX_PATH
Expand All @@ -93,7 +125,40 @@ pushd "${python_build_dir}"
# on Debian/Ubuntu (ARROW-15243).
# - Cannot use build isolation as we want to use specific dependency versions
# (e.g. Numpy, Pandas) on some CI jobs.
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .

# The conda compilers package adds flags for debugging and optimization
# that are unnecessary in a CI context
OLD_CFLAGS=$CFLAGS
OLD_CPPFLAGS=$CPPFLAGS
OLD_CXXFLAGS=$CXXFLAGS
export CFLAGS=
export CPPFLAGS=
export CXXFLAGS=

BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
BUILD_TYPE=${BUILD_TYPE,,} # Meson requires lowercase values

${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv . \
-Csetup-args="-Dbuildtype=${BUILD_TYPE}" \
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
-Ccompile-args="-v" \
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"

export CFLAGS=$OLD_CFLAGS
export CPPFLAGS=$OLD_CPPFLAGS
export CXXFLAGS=$OLD_CXXFLAGS
popd

if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then
Expand Down
6 changes: 5 additions & 1 deletion ci/scripts/python_sdist_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,9 @@ source_dir=${1}/python

pushd "${source_dir}"
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
${PYTHON:-python} setup.py sdist
# Meson dist must be run from a VCS, so initiate a dummy repo
git init .
git add --all .
git commit -m "dummy commit for meson dist"
${PYTHON:-python} -m build --sdist .
Copy link
Member

@raulcd raulcd Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've tried a couple of things to try and fix sdist:

diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh
index 7bea9c3dc1..a8d849183c 100755
--- a/ci/scripts/python_sdist_build.sh
+++ b/ci/scripts/python_sdist_build.sh
@@ -24,8 +24,12 @@ source_dir=${1}/python
 pushd "${source_dir}"
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
 # Meson dist must be run from a VCS, so initiate a dummy repo
-git init .
-git add --all .
-git commit -m "dummy commit for meson dist"
+#git init .
+#git config --global --add safe.directory "${source_dir}"
+#git config --global user.name "Your Name"
+#git config --global user.email your.email@example.com
+#git add --all .
+#git commit -m "dummy commit for meson dist"
+${PYTHON:-python} -m pip install build
 ${PYTHON:-python} -m build --sdist .
 popd

I am unsure whether creating the git repo is necessary (we are already cloning arrow) and build seems missing from this docker image.
The problem now is that it seems to want to build pyarrow (requires build-tools), even though building only the sdist shouldn't require building pyarrow as far as I understand:

+ /arrow-dev/bin/python -m build --sdist .
* Creating isolated environment: venv+pip...
* Installing packages in isolated environment:
  - cython >= 3.1
  - meson-python
  - numpy>=1.25
  - setuptools_scm[toml]>=8
* Getting build dependencies for sdist...
* Installing packages in isolated environment:
  - ninja >= 1.8.2
* Building sdist...
+ meson setup /arrow/python /arrow/python/.mesonpy-f5plfyxb -Dbuildtype=release -Db_ndebug=if-release -Db_vscrt=md --native-file=/arrow/python/.mesonpy-f5plfyxb/meson-python-native-file.ini
The Meson build system
Version: 1.10.0
Source dir: /arrow/python
Build dir: /arrow/python/.mesonpy-f5plfyxb
Build type: native build
Project name: pyarrow
Project version: 23.0.0.dev362+gce85d064a.d20260114

../meson.build:18:0: ERROR: Unknown compiler(s): [['c++'], ['g++'], ['clang++'], ['nvc++'], ['pgc++'], ['icpc'], ['icpx']]
The following exception(s) were encountered:
Running `c++ --version` gave "[Errno 2] No such file or directory: 'c++'"
Running `g++ --version` gave "[Errno 2] No such file or directory: 'g++'"
Running `clang++ --version` gave "[Errno 2] No such file or directory: 'clang++'"
Running `nvc++ --version` gave "[Errno 2] No such file or directory: 'nvc++'"
Running `pgc++ --version` gave "[Errno 2] No such file or directory: 'pgc++'"
Running `icpc --version` gave "[Errno 2] No such file or directory: 'icpc'"
Running `icpx --version` gave "[Errno 2] No such file or directory: 'icpx'"

A full log can be found at /arrow/python/.mesonpy-f5plfyxb/meson-logs/meson-log.txt

ERROR Backend subprocess exited when trying to invoke build_sdist

If I install build-essential it fails locating Arrow CPP because it's basically trying to build pyarrow.

Copy link
Contributor Author

@WillAyd WillAyd Jan 14, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am unsure whether creating the git repo is necessary

It is because building an sdist calls meson dist under the hood, which itself requires to be run from a VCS

The problem now is that it seems to want to build pyarrow

Try adding -Csetup-args="-Dsdist=true" to the invocation. This is something we encountered previously on this PR, so I added that option (noted in the top level python/meson.build file and upstream in mesonbuild/meson-python#647)

popd
4 changes: 0 additions & 4 deletions cpp/cmake_modules/FindSnappyAlt.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,6 @@ if(ARROW_SNAPPY_USE_SHARED)
"${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}")
else()
set(SNAPPY_STATIC_LIB_NAME_BASE "snappy")
if(MSVC)
set(SNAPPY_STATIC_LIB_NAME_BASE
"${SNAPPY_STATIC_LIB_NAME_BASE}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}")
endif()
set(SNAPPY_LIB_NAMES
"${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
Expand Down
Loading
Loading