GPU CMake: Create const param files for available default architectures to be loaded by RTC + Cleanup

GPU/Common/CMakeLists.txt

@@ -62,10 +62,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
                 COMPONENT_NAME GPU
                 LABELS gpu)
   endif()
-  install(FILES ${HDRS_INSTALL} DESTINATION include/GPU)
 endif()
 
-if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
-  install(FILES ${HDRS_INSTALL}
-  DESTINATION include)
-endif()
+install(FILES ${HDRS_INSTALL} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)

GPU/GPUTracking/Base/cuda/CMakeLists.txt

@@ -120,7 +120,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
       ${CMAKE_SOURCE_DIR}/DataFormats/Reconstruction/src
       ${CMAKE_CURRENT_SOURCE_DIR}
     TARGETVARNAME targetName)
-  install(FILES ${HDRS} DESTINATION include/GPU)
 endif()
 
 if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
@@ -132,6 +131,8 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 endif()
 
+install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
+
 target_compile_definitions(${targetName} PRIVATE $<TARGET_PROPERTY:O2::GPUTracking,COMPILE_DEFINITIONS>)
 
 if (onnxruntime_FOUND)

GPU/GPUTracking/Base/hip/CMakeLists.txt

@@ -169,8 +169,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
       ${GPUCA_HIP_SOURCE_DIR}
     TARGETVARNAME targetName)
 
-  install(FILES ${HDRS} DESTINATION include/GPU)
-
 #  o2_add_test(GPUsortHIP NAME test_GPUsortHIP
 #                  SOURCES test/testGPUsortHIP.hip
 #                  PUBLIC_LINK_LIBRARIES O2::GPUCommon hip::host hip::device hip::hipcub roc::rocthrust
@@ -187,6 +185,8 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   include_directories(${GPUCA_HIP_SOURCE_DIR})
 endif()
 
+install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
+
 target_compile_definitions(${targetName} PRIVATE $<TARGET_PROPERTY:O2::GPUTracking,COMPILE_DEFINITIONS>)
 
 if (onnxruntime_FOUND)

GPU/GPUTracking/Base/opencl/CMakeLists.txt

@@ -96,8 +96,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
   # the compile_defitions are not propagated automatically on purpose (they are
   # declared PRIVATE) so we are not leaking them outside of the GPU**
   # directories
-
-  install(FILES ${HDRS} DESTINATION include/GPU)
 endif()
 
 if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
@@ -107,6 +105,8 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   set(targetName ${MODULE})
 endif()
 
+install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
+
 if(OPENCL_ENABLED_SPIRV)
   target_compile_definitions(${targetName} PRIVATE OPENCL_ENABLED_SPIRV)
 endif()

GPU/GPUTracking/CMakeLists.txt

@@ -242,12 +242,13 @@ set(TEMPLATE_HEADER_LIST Base/GPUReconstructionKernelList.template.h
                          Definitions/GPUDefParametersLoad.template.inc)
 set(GENERATED_HEADERS_LIST "")
 
-file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
+set(ON_THE_FLY_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
+file(MAKE_DIRECTORY ${ON_THE_FLY_DIR})
 foreach(TEMPLATE_FILE ${TEMPLATE_HEADER_LIST})
   get_filename_component(OUTPUT_FILE_NAME ${TEMPLATE_FILE} NAME)
   string(REPLACE ".template" "" OUTPUT_FILE_NAME ${OUTPUT_FILE_NAME})
-  file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly/${OUTPUT_FILE_NAME} INPUT ${TEMPLATE_FILE})
-  list(APPEND GENERATED_HEADERS_LIST ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly/${OUTPUT_FILE_NAME})
+  file(GENERATE OUTPUT ${ON_THE_FLY_DIR}/${OUTPUT_FILE_NAME} INPUT ${TEMPLATE_FILE})
+  list(APPEND GENERATED_HEADERS_LIST ${ON_THE_FLY_DIR}/${OUTPUT_FILE_NAME})
 endforeach()
 set(GPUDEFPARAMETERSLBLIST "$<JOIN:$<LIST:TRANSFORM,$<LIST:TRANSFORM,$<TARGET_PROPERTY:O2_GPU_KERNELS,O2_GPU_KERNEL_NAMES>,REPLACE,[^A-Za-z0-9]+,_>,PREPEND,LB_>,\n>\n")
 string(APPEND GPUDEFPARAMETERSLBLIST "$<JOIN:$<LIST:TRANSFORM,$<TARGET_PROPERTY:O2_GPU_KERNELS,O2_GPU_KERNEL_PARAMS>,PREPEND,PAR_>,\n>\n")
@@ -256,14 +257,14 @@ file(GENERATE
      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/GPUDefParametersLoadPrepareBase
      CONTENT ${GPUDEFPARAMETERSLBLIST})
 add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly/GPUDefParametersLoadPrepare.h
-  COMMAND awk "{print(\"#ifndef GPUCA_\" $0 \"\\n#define GPUCA_\" $0 \" 0\\n#endif\")}" ${CMAKE_CURRENT_BINARY_DIR}/GPUDefParametersLoadPrepareBase > ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly/GPUDefParametersLoadPrepare.h
+  OUTPUT ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
+  COMMAND awk "{print(\"#ifndef GPUCA_\" $0 \"\\n#define GPUCA_\" $0 \" 0\\n#endif\")}" ${CMAKE_CURRENT_BINARY_DIR}/GPUDefParametersLoadPrepareBase > ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
   COMMENT "Generating GPUDefParametersLoadPrepare.h"
   DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/GPUDefParametersLoadPrepareBase
   VERBATIM
   COMMAND_EXPAND_LISTS
 )
-list(APPEND GENERATED_HEADERS_LIST ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly/GPUDefParametersLoadPrepare.h)
+list(APPEND GENERATED_HEADERS_LIST ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h)
 
 set(HDRS_INSTALL ${HDRS_INSTALL} ${GENERATED_HEADERS_LIST})
 include(kernels.cmake)
@@ -306,7 +307,7 @@ set(INCDIRS
     ${CMAKE_CURRENT_SOURCE_DIR}/Debug
     ${CMAKE_CURRENT_SOURCE_DIR}/DataCompression
     ${CMAKE_CURRENT_SOURCE_DIR}/TPCClusterFinder
-    ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
+    ${ON_THE_FLY_DIR})
 
 # Main CMake part for O2
 if(ALIGPU_BUILD_TYPE STREQUAL "O2")
@@ -356,12 +357,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
                             HEADERS ${HDRS_CINT_O2} ${HDRS_CINT_O2_ADDITIONAL}
                             LINKDEF GPUTrackingLinkDef_O2.h)
 
-  install(FILES ${HDRS_SRCS} ${HDRS_CINT_O2} ${HDRS_CINT_DATATYPES} ${HDRS_INSTALL}
-          DESTINATION include/GPU)
-  install(DIRECTORY utils
-          DESTINATION include/GPU
-          FILES_MATCHING PATTERN *.h)
-
   o2_add_test_root_macro(Standalone/tools/createGeo.C
                          PUBLIC_LINK_LIBRARIES O2::GPUTracking
                          LABELS its COMPILE_ONLY)
@@ -390,13 +385,15 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   else()
     target_compile_definitions(${targetName} PRIVATE GPUCA_NO_ROOT)
   endif()
-  install(FILES ${HDRS_SRCS} ${HDRS_CINT_O2} ${HDRS_CINT_DATATYPES} ${HDRS_INSTALL}
-          DESTINATION include)
-  install(DIRECTORY utils
-          DESTINATION include
-          FILES_MATCHING PATTERN *.h)
 endif()
 
+install(FILES ${HDRS_SRCS} ${HDRS_CINT_O2} ${HDRS_CINT_DATATYPES} ${HDRS_INSTALL}
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
+install(DIRECTORY utils
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU
+        FILES_MATCHING PATTERN *.h)
+
+
 # GPUReconstructionLibrary needs to know which GPU backends are enabled for proper error messages
 configure_file(Base/GPUReconstructionAvailableBackends.template.h ${CMAKE_CURRENT_BINARY_DIR}/GPUReconstructionAvailableBackends.h)
 set_source_files_properties(Base/GPUReconstructionLibrary.cxx
@@ -410,6 +407,30 @@ target_sources(${targetName}
                FILES ${GENERATED_HEADERS_LIST}
                BASE_DIRS ${CMAKE_CURRENT_BINARY_DIR})
 
+make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch)
+set(GPU_CONST_PARAM_ARCHITECTUES AMPERE TURING VEGA MI2xx)
+set(GPU_CONST_PARAM_FILES "")
+foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES})
+  set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par)
+  add_custom_command(
+    OUTPUT ${PARAMFILE}
+    COMMAND bash -c
+            "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
+            | root -l -b > /dev/null
+    VERBATIM
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch
+    MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C
+    DEPENDS Definitions/GPUDefParametersDefaults.h
+            ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
+            ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc
+    COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}")
+  LIST(APPEND GPU_CONST_PARAM_FILES ${PARAMFILE})
+endforeach()
+
+add_custom_target(${MODULE}_GPU_CONST_PARAM_ARCHS ALL DEPENDS ${GPU_CONST_PARAM_FILES})
+install(FILES ${GPU_CONST_PARAM_FILES} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/arch_param)
+
+
 # Add compile definitions and libraries depending on available optional dependencies
 if(GPUCA_QA)
   message(STATUS "Building GPU QA")

GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h

@@ -22,6 +22,7 @@
 
 // GPU Run Configuration
 #if defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS) // Avoid including for RTC generation besides normal include protection.
+  #define GPUCA_LB_SCAN 512
   // GPU-architecture-dependent default settings
   #if defined(GPUCA_GPUTYPE_MI2xx)
     #define GPUCA_WARP_SIZE 64
@@ -498,11 +499,11 @@
   #define GPUCA_LB_GPUTPCNNClusterizerKernels_publishClass1Regression GPUCA_LB_GPUTPCNNClusterizerKernels
   #define GPUCA_LB_GPUTPCNNClusterizerKernels_publishClass2Regression GPUCA_LB_GPUTPCNNClusterizerKernels
 
-  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanStart GPUCA_THREAD_COUNT_SCAN
-  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanUp GPUCA_THREAD_COUNT_SCAN
-  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanTop GPUCA_THREAD_COUNT_SCAN
-  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanDown GPUCA_THREAD_COUNT_SCAN
-  #define GPUCA_LB_GPUTPCCFStreamCompaction_compactDigits GPUCA_THREAD_COUNT_SCAN
+  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanStart GPUCA_LB_SCAN
+  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanUp GPUCA_LB_SCAN
+  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanTop GPUCA_LB_SCAN
+  #define GPUCA_LB_GPUTPCCFStreamCompaction_scanDown GPUCA_LB_SCAN
+  #define GPUCA_LB_GPUTPCCFStreamCompaction_compactDigits GPUCA_LB_SCAN
   #define GPUCA_LB_GPUTPCCompressionGatherKernels_unbuffered GPUCA_LB_COMPRESSION_GATHER
   #define GPUCA_LB_GPUTPCCompressionGatherKernels_buffered32 GPUCA_LB_COMPRESSION_GATHER
   #define GPUCA_LB_GPUTPCCompressionGatherKernels_buffered64 GPUCA_LB_COMPRESSION_GATHER

GPU/GPUTracking/Interface/CMakeLists.txt

@@ -46,4 +46,4 @@ target_include_directories(${targetName}
 target_compile_definitions(${targetName} PRIVATE $<TARGET_PROPERTY:O2::GPUTracking,COMPILE_DEFINITIONS>)
 
 
-install(FILES ${HDRS} DESTINATION include/GPU)
+install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)

GPU/GPUTracking/Standalone/CMakeLists.txt

@@ -13,6 +13,7 @@
 cmake_minimum_required(VERSION 3.27 FATAL_ERROR)
 project(GPUTrackingStandalone)
 include(FeatureSummary)
+include(GNUInstallDirs)
 set(CMAKE_INSTALL_MESSAGE LAZY)
 
 set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}")
@@ -241,3 +242,4 @@ install(TARGETS ca TPCFastTransformation standalone_support)
 install(FILES "cmake/makefile" DESTINATION "${CMAKE_INSTALL_PREFIX}")
 install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${O2_DIR} ${CMAKE_INSTALL_PREFIX}/src)")
 install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_BINARY_DIR}/config.cmake ${CMAKE_INSTALL_PREFIX}/config.cmake)")
+install(DIRECTORY tools DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU)

GPU/GPUTracking/Standalone/tools/dumpGPUDefParam.C

@@ -12,27 +12,24 @@
 /// \file dumpGPUDefParam.C
 /// \author David Rohr
 
-// Run e.g. as:
-// ROOT_INCLUDE_PATH="`pwd`/include" root -l -q -b src/GPU/GPUTracking/Standalone/tools/dumpGPUDefParam.C'()'
+// Run e.g. as (replacing [FILE] and [OUTPUT]:
+// echo -e '#define PARAMETER_FILE "[FILE]]"\ngInterpreter->AddIncludePath("'`pwd`'/include/GPU");\n.x share/GPU/tools/dumpGPUDefParam.C("[OUTPUT]")\n.q\n' | root -l -b
+// To dump the defaults for AMPERE architecture, run
+// echo -e '#define GPUCA_GPUTYPE_AMPERE\n#define PARAMETER_FILE "GPUDefParametersDefaults.h"\ngInterpreter->AddIncludePath("'`pwd`'/include/GPU");\n.x share/GPU/tools/dumpGPUDefParam.C("default_AMPERE.par")\n.q\n' | root -l -b
 
-// Logic for testing to load the default parameters
-/* #define GPUCA_GPUCODE
-#define GPUCA_GPUTYPE_AMPERE
-#define GPUCA_MAXN 40
-#define GPUCA_ROW_COUNT 152
-#define GPUCA_TPC_COMP_CHUNK_SIZE 1024
-#include "GPUDefParametersConstants.h"
-#include "GPUDefParametersDefaults.h" */
+#ifndef PARAMETER_FILE
+#error Must provide the PARAMETER_FILE as preprocessor define, e.g. -DHEADER_TO_INCLUDE='"GPUDefParametersDefaults.h"'
+#endif
 
-// Alternatively, logic to load file that sets GPUDefParameters
-#include "testParam.h"
+#define GPUCA_GPUCODE
+#include PARAMETER_FILE
 
 #include "GPUDefParametersLoad.inc"
-void dumpGPUDefParam()
+void dumpGPUDefParam(const char* outputfile = "parameters.out")
 {
   auto param = o2::gpu::internal::GPUDefParametersLoad();
-  printf("Loaded params:\n%s", o2::gpu::internal::GPUDefParametersExport(param, false).c_str());
-  FILE* fp = fopen("parameters.out", "w+b");
+  printf("Loaded params:\n%s\nWriting them to %s\n", o2::gpu::internal::GPUDefParametersExport(param, false).c_str(), outputfile);
+  FILE* fp = fopen(outputfile, "w+b");
   fwrite(&param, 1, sizeof(param), fp);
   fclose(fp);
 }

GPU/GPUTracking/display/CMakeLists.txt

@@ -133,8 +133,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
 
   target_compile_definitions(${targetName} PRIVATE GPUCA_BUILD_EVENT_DISPLAY_GLFW GPUCA_DISPLAY_GL3W GPUCA_DISPLAY_OPENGL_CORE)
 
-  install(FILES ${HDRS} ${HDRS_INSTALL} DESTINATION include/GPU)
-
   o2_add_executable(field-uniform-exporter
           TARGETVARNAME exporterName
           COMPONENT_NAME gpu
@@ -151,12 +149,14 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   target_link_libraries(${targetName} PUBLIC O2::GPUTracking)
   install(TARGETS ${MODULE})
 
-  install(DIRECTORY filterMacros/ DESTINATION displayTrackFilter FILES_MATCHING PATTERN "*.C")
+  install(DIRECTORY filterMacros/ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/displayTrackFilter FILES_MATCHING PATTERN "*.C")
   get_property(GPU_DISPLAY_INCLUDE_PATH DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
   configure_file(filterMacros/setinclude.sh.in setinclude.sh @ONLY)
-  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/setinclude.sh PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE DESTINATION displayTrackFilter)
+  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/setinclude.sh PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/displayTrackFilter)
 endif()
 
+install(FILES ${HDRS} ${HDRS_INSTALL} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
+
 target_compile_definitions(${targetName} PRIVATE $<TARGET_PROPERTY:O2::GPUTracking,COMPILE_DEFINITIONS>)
 
 message(STATUS "Building GPU Event Display (Vulkan ${GPUCA_EVENT_DISPLAY_VULKAN}, Wayland ${GPUCA_EVENT_DISPLAY_WAYLAND}, Freetype ${GPUCA_EVENT_DISPLAY_FREETYPE}, Fontconfig ${Fontconfig_FOUND}, Qt ${GPUCA_EVENT_DISPLAY_QT})")

GPU/TPCFastTransformation/CMakeLists.txt

@@ -64,7 +64,6 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
                             HEADERS ${HDRS_CINT_O2}
                             LINKDEF TPCFastTransformationLinkDef_O2.h)
 
-  install(FILES ${HDRS_CINT_O2} DESTINATION include/GPU)
   file(COPY ${HDRS_CINT_O2} DESTINATION ${CMAKE_BINARY_DIR}/stage/include/GPU)
 
   o2_add_test(${MODULE}
@@ -115,6 +114,8 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
                            LABELS gpu tpc)
   endforeach()
 
+  install(FILES macro/TPCFastTransformInit.C
+          DESTINATION share/macro/)
 endif()
 
 if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
@@ -123,8 +124,7 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone")
   target_include_directories(${targetName} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 endif()
 
-install (FILES macro/TPCFastTransformInit.C
-         DESTINATION share/macro/)
+install(FILES ${HDRS_CINT_O2} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)
 
 #
 # FIXME: this one is misplaced : it depends (at least) on TPCSimulation which is

GPU/Utils/CMakeLists.txt

@@ -31,5 +31,6 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
                             HEADERS ${HDRS_CINT}
                             LINKDEF GPUUtilsLinkDef.h)
 
-  install(FILES ${HDRS_CINT} ${HDRS_INSTALL} DESTINATION include/GPU)
 endif()
+
+install(FILES ${HDRS_CINT} ${HDRS_INSTALL} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU)