Skip to content

[Feature](paimon) Integrate paimon-cpp library for native Paimon table support #60194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xylaaaaa wants to merge 10 commits into
base: master
Choose a base branch
from
Open

[Feature](paimon) Integrate paimon-cpp library for native Paimon table support #60194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
281d993
feat: integrate paimon-cpp library and add demo tool
xylaaaaa Jan 12, 2026
7ec8350
fix
xylaaaaa Jan 17, 2026
1b182fe
fix
xylaaaaa Jan 17, 2026
25cdb7b
fix
xylaaaaa Jan 19, 2026
b3c1054
fix
xylaaaaa Jan 21, 2026
1f5a84d
Merge remote-tracking branch 'origin/master' into feature/paimon-cpp-…
xylaaaaa Jan 22, 2026
0abad8b
fix
xylaaaaa Jan 22, 2026
5936baa
fix
morningman Jan 23, 2026
2096770
fix
morningman Jan 23, 2026
1c9b3eb
fix
morningman Jan 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 110 additions & 0 deletions be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,21 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}"
option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF)
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")

option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
set(PAIMON_USE_EXTERNAL_ORC OFF CACHE BOOL "Use Doris ORC for Paimon C++")

# Allow env to override when reconfiguring (avoid picking /usr/local).
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
endif()
if (DEFINED ENV{PAIMON_USE_EXTERNAL_ORC})
set(PAIMON_USE_EXTERNAL_ORC "$ENV{PAIMON_USE_EXTERNAL_ORC}" CACHE BOOL "" FORCE)
endif()

set(CMAKE_SKIP_RPATH TRUE)
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
Expand Down Expand Up @@ -207,6 +222,9 @@ SET(CONTRIB_PATH "${PROJECT_SOURCE_DIR}/../contrib")
# Out of source build need to set the binary dir
add_subdirectory(${CONTRIB_PATH}/apache-orc ${PROJECT_BINARY_DIR}/apache-orc EXCLUDE_FROM_ALL)
target_compile_options(orc PRIVATE -Wno-implicit-fallthrough -w)
if(TARGET orc AND NOT TARGET orc::orc)
add_library(orc::orc ALIAS orc)
endif()

option(BUILD_STATIC_LIBRARIES "Build static libraries" ON)
option(BUILD_SHARED_LIBRARIES "Build shared libraries" OFF)
Expand Down Expand Up @@ -544,6 +562,91 @@ set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
)

if (ENABLE_PAIMON_CPP)
if (NOT PAIMON_HOME)
set(_paimon_default_home "${THIRDPARTY_DIR}/paimon-cpp")
if (EXISTS "${_paimon_default_home}/lib/cmake/Paimon/PaimonConfig.cmake")
set(PAIMON_HOME "${_paimon_default_home}" CACHE PATH "" FORCE)
endif()
unset(_paimon_default_home)
endif()
if (NOT PAIMON_HOME)
message(FATAL_ERROR "ENABLE_PAIMON_CPP=ON but PAIMON_HOME is not set")
endif()
set(Paimon_DIR "${PAIMON_HOME}/lib/cmake/Paimon" CACHE PATH "" FORCE)
find_package(Paimon CONFIG REQUIRED NO_DEFAULT_PATH)
include_directories("${PAIMON_HOME}/include")
# Link glog after paimon static libs to satisfy RawLog__ in static link.
list(REMOVE_ITEM COMMON_THIRDPARTY glog)

# Paimon static library dependencies
set(paimon_fmt_lib "${PAIMON_HOME}/lib64/paimon_deps/libfmtd.a")
if (NOT EXISTS "${paimon_fmt_lib}")
set(paimon_fmt_lib "${PAIMON_HOME}/lib64/paimon_deps/libfmt.a")
endif()
add_library(paimon_fmt STATIC IMPORTED)
set_target_properties(paimon_fmt PROPERTIES IMPORTED_LOCATION "${paimon_fmt_lib}")
set(paimon_tbb_lib "${PAIMON_HOME}/lib64/paimon_deps/libtbb_debug.a")
if (NOT EXISTS "${paimon_tbb_lib}")
set(paimon_tbb_lib "${PAIMON_HOME}/lib64/paimon_deps/libtbb.a")
endif()
add_library(paimon_tbb STATIC IMPORTED)
set_target_properties(paimon_tbb PROPERTIES IMPORTED_LOCATION "${paimon_tbb_lib}")
add_library(paimon_roaring STATIC IMPORTED)
set_target_properties(paimon_roaring PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libroaring_bitmap.a")
add_library(paimon_xxhash STATIC IMPORTED)
set_target_properties(paimon_xxhash PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libxxhash.a")
add_library(paimon_arrow_dataset STATIC IMPORTED)
set_target_properties(paimon_arrow_dataset PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libarrow_dataset.a")
add_library(paimon_arrow_acero STATIC IMPORTED)
set_target_properties(paimon_arrow_acero PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libarrow_acero.a")
add_library(paimon_arrow STATIC IMPORTED)
set_target_properties(paimon_arrow PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libarrow.a")
set(paimon_arrow_bundled_deps_lib "${PAIMON_HOME}/lib64/paimon_deps/libarrow_bundled_dependencies.a")
if (EXISTS "${paimon_arrow_bundled_deps_lib}")
add_library(paimon_arrow_bundled_dependencies STATIC IMPORTED)
set_target_properties(paimon_arrow_bundled_dependencies PROPERTIES IMPORTED_LOCATION "${paimon_arrow_bundled_deps_lib}")
endif()
add_library(paimon_parquet STATIC IMPORTED)
set_target_properties(paimon_parquet PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/libparquet.a")
if(NOT PAIMON_USE_EXTERNAL_ORC)
add_library(paimon_orc STATIC IMPORTED)
set_target_properties(paimon_orc PROPERTIES IMPORTED_LOCATION "${PAIMON_HOME}/lib64/paimon_deps/liborc.a")
endif()
Comment on lines +582 to +615
Copy link

Copilot AI Jan 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The imported paimon dependency libraries (fmt/tbb fallbacks, roaring/xxhash/arrow/parquet/orc) are not validated after selecting paths. If any file is missing, CMake will fail later with a less actionable error. Consider adding explicit EXISTS checks and a clear FATAL_ERROR listing the missing artifacts under PAIMON_HOME, especially after the fallback selection for fmt/tbb.

Copilot uses AI. Check for mistakes.

set(PAIMON_CPP_FACTORY_TARGETS
paimon_static
paimon_parquet_file_format_static
paimon_file_index_static
paimon_global_index_static
paimon_local_file_system_static
)
if(PAIMON_USE_EXTERNAL_ORC)
list(APPEND PAIMON_CPP_FACTORY_TARGETS paimon_orc_file_format_static)
endif()
set(PAIMON_CPP_DEP_TARGETS
paimon_fmt
paimon_tbb
paimon_roaring
paimon_xxhash
paimon_parquet
paimon_arrow_dataset
paimon_arrow_acero
paimon_arrow
)
if (TARGET paimon_arrow_bundled_dependencies)
list(APPEND PAIMON_CPP_DEP_TARGETS paimon_arrow_bundled_dependencies)
endif()
list(APPEND PAIMON_CPP_DEP_TARGETS glog)
# Force-link factory libraries so static init registers file formats.
list(APPEND COMMON_THIRDPARTY
-Wl,--whole-archive
${PAIMON_CPP_FACTORY_TARGETS}
-Wl,--no-whole-archive
${PAIMON_CPP_DEP_TARGETS})
message(STATUS "Paimon C++ enabled: ${PAIMON_HOME}")
endif()

if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
add_library(hadoop_hdfs STATIC IMPORTED)
set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
Expand Down Expand Up @@ -571,6 +674,13 @@ endif()
if (absl_FOUND)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
absl::cord
absl::cord_internal
absl::cordz_functions
absl::cordz_info
absl::cordz_update_scope
absl::cordz_update_tracker
absl::crc_cord_state
absl::flags
absl::random_random
absl::spinlock_wait
Expand Down
31 changes: 31 additions & 0 deletions be/src/tools/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,34 @@ add_custom_command(TARGET packed_file_tool POST_BUILD
COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:packed_file_tool>.dbg $<TARGET_FILE:packed_file_tool>
)
endif()

if (ENABLE_PAIMON_CPP)
add_executable(paimon_cpp_demo
paimon_cpp_demo.cpp
)

pch_reuse(paimon_cpp_demo)

set_target_properties(paimon_cpp_demo PROPERTIES ENABLE_EXPORTS 1)

if (COMPILER_CLANG)
target_compile_options(paimon_cpp_demo PRIVATE
-Wno-implicit-int-conversion
-Wno-shorten-64-to-32
)
endif()

target_link_libraries(paimon_cpp_demo
${DORIS_LINK_LIBS}
)

install(TARGETS paimon_cpp_demo DESTINATION ${OUTPUT_DIR}/lib/)

if (NOT OS_MACOSX)
add_custom_command(TARGET paimon_cpp_demo POST_BUILD
COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $<TARGET_FILE:paimon_cpp_demo> $<TARGET_FILE:paimon_cpp_demo>.dbg
COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:paimon_cpp_demo>
COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:paimon_cpp_demo>.dbg $<TARGET_FILE:paimon_cpp_demo>
)
endif()
endif()
Loading