Skip to content

Commit 6de9ef3

Browse files
haochengxiahaochengxia
committed
Fix ci build wheels
1 parent 90a9fa0 commit 6de9ef3

File tree

5 files changed

+292
-15
lines changed

5 files changed

+292
-15
lines changed

‎CMakeLists.txt‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ project(libCacheSim-python)
33
set(DESCRIPTION "The libCacheSim Python Package")
44
set(PROJECT_WEB "https://docs.libcachesim.com/python")
55

6+
set(CXX_FLAGS "-Wno-cast-user-defined -Wno-array-bounds")
7+
68
# Options from the libCacheSim library
79
option(ENABLE_GLCACHE "enable group-learned cache" ON)
810
option(ENABLE_LRB "enable LRB" ON)
@@ -38,8 +40,25 @@ endif()
3840
# Build libCacheSim first
3941
set(LIBCACHESIM_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/libCacheSim/build")
4042
message(STATUS "Building libCacheSim...")
43+
44+
# Set compiler environment variables if available
45+
set(CMAKE_ARGS "")
46+
if(DEFINED ENV{CC})
47+
list(APPEND CMAKE_ARGS "-DCMAKE_C_COMPILER=$ENV{CC}")
48+
endif()
49+
if(DEFINED ENV{CXX})
50+
list(APPEND CMAKE_ARGS "-DCMAKE_CXX_COMPILER=$ENV{CXX}")
51+
endif()
52+
53+
# Add compiler flags to disable specific checks
54+
list(APPEND CMAKE_ARGS "-DCMAKE_C_FLAGS=-fPIC -fno-strict-overflow -fno-strict-aliasing")
55+
list(APPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS=-fPIC -fno-strict-overflow -fno-strict-aliasing ${CXX_FLAGS}")
56+
list(APPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS_DEBUG=-g ${CXX_FLAGS}")
57+
list(APPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS_RELEASE=-O3 ${CXX_FLAGS}")
58+
list(APPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=-O2 -g -DNDEBUG ${CXX_FLAGS}")
59+
4160
execute_process(
42-
COMMAND ${CMAKE_COMMAND} -S ${LIBCACHESIM_SOURCE_DIR} -B ${LIBCACHESIM_BUILD_DIR} -G Ninja -DENABLE_LRB=${ENABLE_LRB} -DENABLE_GLCACHE=${ENABLE_GLCACHE} -DENABLE_3L_CACHE=${ENABLE_3L_CACHE} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
61+
COMMAND ${CMAKE_COMMAND} -S ${LIBCACHESIM_SOURCE_DIR} -B ${LIBCACHESIM_BUILD_DIR} -G Ninja -DENABLE_LRB=${ENABLE_LRB} -DENABLE_GLCACHE=${ENABLE_GLCACHE} -DENABLE_3L_CACHE=${ENABLE_3L_CACHE} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ${CMAKE_ARGS}
4362
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
4463
RESULT_VARIABLE CMAKE_CONFIG_RESULT
4564
)
@@ -75,10 +94,10 @@ else()
7594
message(FATAL_ERROR "export_vars.cmake not found at ${EXPORT_FILE}. Please build the main project first (e.g. cd .. && cmake -G Ninja -B build)")
7695
endif()
7796

78-
# Force enable -fPIC
97+
# Force enable -fPIC and disable specific checks
7998
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
80-
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
81-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
99+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-overflow -fno-strict-aliasing")
100+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fno-strict-overflow -fno-strict-aliasing ${CXX_FLAGS}")
82101

83102
project(libCacheSim-python VERSION "${LIBCACHESIM_VERSION}")
84103

‎pyproject.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,12 @@ environment = { LCS_BUILD_DIR = "{project}/src/libCacheSim/build", MACOSX_DEPLOY
8686
test-command = "python -c 'import libcachesim; print(\"Import successful\")'"
8787

8888
[tool.cibuildwheel.linux]
89-
before-all = "yum install -y yum-utils && yum-config-manager --set-enabled crb && yum install -y ninja-build cmake libzstd-devel glib2-devel xgboost-devel lightgbm-devel"
90-
before-build = "pip install pybind11 && git submodule update --init --recursive"
89+
before-all = "yum install -y yum-utils && yum-config-manager --set-enabled crb && yum install -y git && git submodule update --init --recursive && bash scripts/install_deps.sh"
90+
before-build = "pip install pybind11 && rm -rf src/libCacheSim/build"
9191

9292
[tool.cibuildwheel.macos]
93-
before-all = "brew install glib google-perftools argp-standalone xxhash llvm wget cmake ninja zstd xgboost lightgbm"
94-
before-build = "pip install pybind11 && git submodule update --init --recursive"
93+
before-all = "brew install git && git submodule update --init --recursive && bash scripts/install_deps.sh"
94+
before-build = "pip install pybind11 && rm -rf src/libCacheSim/build"
9595

9696
[tool.ruff]
9797
# Allow lines to be as long as 120.

‎scripts/install.sh‎

Lines changed: 82 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,36 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
function usage() {
5+
echo "Usage: $0 [options]"
6+
echo "Options:"
7+
echo " -h, --help Show this help message"
8+
echo " -b, --build-wheels Build the Python wheels"
9+
exit 1
10+
}
11+
# Parse command line arguments
12+
BUILD_WHEELS=0
13+
14+
while [[ $# -gt 0 ]]; do
15+
case $1 in
16+
-h|--help)
17+
usage
18+
;;
19+
-b|--build-wheels)
20+
BUILD_WHEELS=1
21+
shift
22+
;;
23+
*)
24+
echo "Unknown option: $1"
25+
usage
26+
;;
27+
esac
28+
done
29+
30+
# For submodule update
131
git submodule update --init --recursive
2-
# Sync submodules
332
git submodule update --recursive --remote
433

5-
6-
# Now build and install the Python binding
7-
echo "Building Python binding..."
8-
echo "Sync python version..."
934
python scripts/sync_version.py
1035
python -m pip install -e . -vvv
1136

@@ -15,4 +40,55 @@ python -c "import libcachesim"
1540

1641
# Run tests
1742
python -m pip install pytest
18-
python -m pytest tests
43+
python -m pytest tests
44+
45+
# Build wheels if requested
46+
if [[ $BUILD_WHEELS -eq 1 ]]; then
47+
echo "--- Building Python wheels for distribution ---"
48+
49+
# --- Environment and dependency checks ---
50+
echo "Checking dependencies: python3, pip, docker, cibuildwheel..."
51+
52+
if ! command -v python3 &> /dev/null; then
53+
echo "Error: python3 is not installed. Please install it and run this script again."
54+
exit 1
55+
fi
56+
57+
if ! python3 -m pip --version &> /dev/null; then
58+
echo "Error: pip for python3 is not available. Please install it."
59+
exit 1
60+
fi
61+
62+
if ! command -v docker &> /dev/null; then
63+
echo "Error: docker is not installed. Please install it and ensure the docker daemon is running."
64+
exit 1
65+
fi
66+
67+
# Check if user can run docker without sudo, otherwise use sudo
68+
SUDO_CMD=""
69+
if ! docker ps &> /dev/null; then
70+
echo "Warning: Current user cannot run docker. Trying with sudo."
71+
if sudo docker ps &> /dev/null; then
72+
SUDO_CMD="sudo"
73+
else
74+
echo "Error: Failed to run docker, even with sudo. Please check your docker installation and permissions."
75+
exit 1
76+
fi
77+
fi
78+
79+
if ! python3 -m cibuildwheel --version &> /dev/null; then
80+
echo "cibuildwheel not found, installing..."
81+
python3 -m pip install cibuildwheel
82+
fi
83+
84+
echo "Dependency check completed."
85+
86+
# --- Run cibuildwheel ---
87+
# The project to build is specified as an argument.
88+
# cibuildwheel should be run from the repository root.
89+
# The output directory will be 'wheelhouse/' by default.
90+
echo "Starting the wheel build process for Linux..."
91+
${SUDO_CMD} python3 -m cibuildwheel --platform linux .
92+
93+
echo "Build process completed successfully. Wheels are in the 'wheelhouse' directory."
94+
fi

‎scripts/install_deps.sh‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
# Colors for output
6+
RED='\033[0;31m'
7+
GREEN='\033[0;32m'
8+
YELLOW='\033[1;33m'
9+
NC='\033[0m' # No Color
10+
11+
# Logging functions
12+
log_info() {
13+
echo -e "${GREEN}[INFO]${NC} $1"
14+
}
15+
16+
log_warn() {
17+
echo -e "${YELLOW}[WARN]${NC} $1"
18+
}
19+
20+
log_error() {
21+
echo -e "${RED}[ERROR]${NC} $1" >&2
22+
}
23+
24+
# Install and configure g++ version
25+
install_gcc() {
26+
log_info "Installing and configuring g++ version..."
27+
28+
# Install gcc-c++ and devtoolset if available
29+
if yum search devtoolset 2>/dev/null | grep -q devtoolset; then
30+
log_info "Installing devtoolset for newer g++ version..."
31+
yum install -y centos-release-scl
32+
yum install -y devtoolset-11-gcc-c++
33+
34+
# Set environment variables globally for this script
35+
export CC=/opt/rh/devtoolset-11/root/usr/bin/gcc
36+
export CXX=/opt/rh/devtoolset-11/root/usr/bin/g++
37+
export PATH=/opt/rh/devtoolset-11/root/usr/bin:$PATH
38+
export LD_LIBRARY_PATH=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:$LD_LIBRARY_PATH
39+
40+
# Create symlinks to make it the default
41+
ln -sf /opt/rh/devtoolset-11/root/usr/bin/gcc /usr/local/bin/gcc
42+
ln -sf /opt/rh/devtoolset-11/root/usr/bin/g++ /usr/local/bin/g++
43+
ln -sf /opt/rh/devtoolset-11/root/usr/bin/gcc /usr/local/bin/cc
44+
ln -sf /opt/rh/devtoolset-11/root/usr/bin/g++ /usr/local/bin/c++
45+
46+
# Add to PATH permanently
47+
echo 'export PATH=/opt/rh/devtoolset-11/root/usr/bin:$PATH' >> /etc/environment
48+
echo 'export LD_LIBRARY_PATH=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:$LD_LIBRARY_PATH' >> /etc/environment
49+
50+
log_info "Using g++ from devtoolset-11: $(g++ --version | head -n1)"
51+
else
52+
log_info "Using system g++: $(g++ --version | head -n1)"
53+
fi
54+
}
55+
56+
# Install CMake
57+
install_cmake() {
58+
log_info "Installing CMake..."
59+
local cmake_version="3.31.0"
60+
local cmake_dir="${HOME}/software/cmake"
61+
62+
pushd /tmp/ >/dev/null
63+
if [[ ! -f "cmake-${cmake_version}-linux-x86_64.sh" ]]; then
64+
wget "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh"
65+
fi
66+
67+
mkdir -p "${cmake_dir}" 2>/dev/null || true
68+
bash "cmake-${cmake_version}-linux-x86_64.sh" --skip-license --prefix="${cmake_dir}"
69+
70+
# Add to shell config files if not already present
71+
for shell_rc in "${HOME}/.bashrc" "${HOME}/.zshrc"; do
72+
# trunk-ignore(shellcheck/SC2016)
73+
if [[ -f ${shell_rc} ]] && ! grep -q 'PATH=$HOME/software/cmake/bin:$PATH' "${shell_rc}"; then
74+
# trunk-ignore(shellcheck/SC2016)
75+
echo 'export PATH=$HOME/software/cmake/bin:$PATH' >>"${shell_rc}"
76+
fi
77+
done
78+
79+
# Source the updated PATH
80+
export PATH="${cmake_dir}/bin:${PATH}"
81+
popd >/dev/null
82+
}
83+
84+
# Install Zstd from source
85+
install_zstd() {
86+
log_info "Installing Zstd from source..."
87+
local zstd_version="1.5.0"
88+
89+
pushd /tmp/ >/dev/null
90+
if [[ ! -f "zstd-${zstd_version}.tar.gz" ]]; then
91+
wget "https://github.com/facebook/zstd/releases/download/v${zstd_version}/zstd-${zstd_version}.tar.gz"
92+
tar xf "zstd-${zstd_version}.tar.gz"
93+
fi
94+
pushd "zstd-${zstd_version}/build/cmake/" >/dev/null
95+
mkdir -p _build
96+
pushd _build >/dev/null
97+
cmake -G Ninja ..
98+
ninja
99+
ninja install
100+
popd >/dev/null
101+
popd >/dev/null
102+
popd >/dev/null
103+
}
104+
105+
# Install XGBoost from source
106+
install_xgboost() {
107+
log_info "Installing XGBoost from source..."
108+
pushd /tmp/ >/dev/null
109+
if [[ ! -d "xgboost" ]]; then
110+
git clone --recursive https://github.com/dmlc/xgboost
111+
fi
112+
pushd xgboost >/dev/null
113+
mkdir -p build
114+
pushd build >/dev/null
115+
cmake -G Ninja ..
116+
ninja
117+
ninja install
118+
popd >/dev/null
119+
popd >/dev/null
120+
popd >/dev/null
121+
}
122+
123+
# Install LightGBM from source
124+
install_lightgbm() {
125+
log_info "Installing LightGBM from source..."
126+
pushd /tmp/ >/dev/null
127+
if [[ ! -d "LightGBM" ]]; then
128+
git clone --recursive https://github.com/microsoft/LightGBM
129+
fi
130+
pushd LightGBM >/dev/null
131+
mkdir -p build
132+
pushd build >/dev/null
133+
cmake -G Ninja ..
134+
ninja
135+
ninja install
136+
popd >/dev/null
137+
popd >/dev/null
138+
popd >/dev/null
139+
}
140+
141+
# Detect OS and install dependencies
142+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
143+
log_info "Detected Linux system, installing dependencies via yum..."
144+
145+
# Enable EPEL repository
146+
yum install -y epel-release
147+
148+
# Enable PowerTools/CRB repository
149+
if yum repolist | grep -q powertools; then
150+
yum-config-manager --set-enabled powertools
151+
elif yum repolist | grep -q crb; then
152+
yum-config-manager --set-enabled crb
153+
fi
154+
155+
# Install development tools
156+
yum groupinstall -y "Development Tools"
157+
yum install -y glib2-devel google-perftools-devel
158+
yum install -y ninja-build git wget
159+
160+
# Install and configure g++ version
161+
install_gcc
162+
163+
# Install CMake
164+
install_cmake
165+
166+
# Install dependencies from source
167+
install_zstd
168+
install_xgboost
169+
install_lightgbm
170+
171+
elif [[ "$OSTYPE" == "darwin"* ]]; then
172+
log_info "Detected macOS system, installing dependencies via brew..."
173+
174+
# Install basic dependencies via Homebrew
175+
brew install glib google-perftools argp-standalone xxhash llvm wget cmake ninja zstd xgboost lightgbm
176+
177+
else
178+
log_error "Unsupported operating system: $OSTYPE"
179+
exit 1
180+
fi
181+
182+
log_info "Dependencies installation completed!"

‎src/libCacheSim‎

0 commit comments

Comments
 (0)