GPU: Make some of the optimizations AMD did for the neighbors finder for MI50 optional

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstruction.cxx

@@ -263,7 +263,7 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice()
   }
   if (mProcessingSettings.deterministicGPUReconstruction) {
 #ifndef GPUCA_DETERMINISTIC_MODE
-    GPUError("Warning, deterministicGPUReconstruction needs GPUCA_DETERMINISTIC_MODE for being fully deterministic, without only most indeterminism by concurrency is removed, but floating point effects remain!");
+    GPUError("WARNING, deterministicGPUReconstruction needs GPUCA_DETERMINISTIC_MODE for being fully deterministic, without only most indeterminism by concurrency is removed, but floating point effects remain!");
 #endif
     mProcessingSettings.overrideClusterizerFragmentLen = TPC_MAX_FRAGMENT_LEN_GPU;
     param().rec.tpc.nWaysOuter = true;
@@ -274,6 +274,10 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice()
       mProcessingSettings.createO2Output = 1;
     }
     mProcessingSettings.rtc.deterministic = 1;
+  } else {
+#ifdef GPUCA_DETERMINISTIC_MODE
+    GPUError("WARNING, compiled with GPUCA_DETERMINISTIC_MODE but deterministicGPUReconstruction not set, only compile-time determinism and deterministic math enforced, not fully deterministic!");
+#endif
   }
   if (mProcessingSettings.deterministicGPUReconstruction && mProcessingSettings.debugLevel >= 6) {
     mProcessingSettings.nTPCClustererLanes = 1;

GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h

@@ -516,6 +516,12 @@
   #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP
     #define GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP 6
   #endif
+  #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL
+    #define GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL 4
+  #endif
+  #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED
+    #define GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED 1
+  #endif
   #ifndef GPUCA_PAR_TRACKLET_SELECTOR_HITS_REG_SIZE
     #define GPUCA_PAR_TRACKLET_SELECTOR_HITS_REG_SIZE 12
   #endif
@@ -544,6 +550,12 @@
   #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP
     #define GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP 0
   #endif
+  #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL
+    #define GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL 0
+  #endif
+  #ifndef GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED
+    #define GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED 0
+  #endif
   #ifndef GPUCA_PAR_TRACKLET_SELECTOR_HITS_REG_SIZE
     #define GPUCA_PAR_TRACKLET_SELECTOR_HITS_REG_SIZE 0
   #endif

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -284,7 +284,7 @@ AddOption(allocDebugLevel, int32_t, 0, "allocDebug", 0, "Some debug output for m
 AddOption(debugMask, int32_t, 262143, "", 0, "Mask for debug output dumps to file")
 AddOption(serializeGPU, int8_t, 0, "", 0, "Synchronize after each kernel call (bit 1) and DMA transfer (bit 2) and identify failures")
 AddOption(recoTaskTiming, bool, 0, "", 0, "Perform summary timing after whole reconstruction tasks")
-AddOption(deterministicGPUReconstruction, int32_t, -1, "", 0, "Make CPU and GPU debug output comparable (sort / skip concurrent parts), -1 = automatic if debugLevel >= 6")
+AddOption(deterministicGPUReconstruction, int32_t, -1, "", 0, "Make CPU and GPU debug output comparable (sort / skip concurrent parts), -1 = automatic if debugLevel >= 6", def(1))
 AddOption(showOutputStat, bool, false, "", 0, "Print some track output statistics")
 AddOption(runCompressionStatistics, bool, false, "compressionStat", 0, "Run statistics and verification for cluster compression")
 AddOption(resetTimers, int8_t, 1, "", 0, "Reset timers every event")

GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx

@@ -75,7 +75,7 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh
     return;
   }
 
-  static constexpr uint32_t UNROLL_GLOBAL = 4;
+  static constexpr uint32_t UNROLL_GLOBAL = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL > 1 ? GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL : 1;
   static_assert(GPUCA_MAXN % UNROLL_GLOBAL == 0);
   static constexpr uint32_t MAX_SHARED = GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP;
   static constexpr uint32_t MAX_GLOBAL = (MAX_SHARED < GPUCA_MAXN) ? (((GPUCA_MAXN - MAX_SHARED - 1) / UNROLL_GLOBAL + 1) * UNROLL_GLOBAL) : 0;
@@ -173,25 +173,25 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh
       }
     }
 
-    if constexpr (MAX_SHARED > 0) { // init the rest of the shared array
+    if constexpr (MAX_SHARED > 0 && GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED) { // init the rest of the shared array
       for (uint32_t iUp = nNeighUp; iUp < MAX_SHARED; iUp++) {
         s.mA1[iUp][iThread] = -1.e10f;
         s.mA2[iUp][iThread] = -1.e10f;
         s.mB[iUp][iThread] = (calink)-1;
       }
     }
 
-    const uint32_t Nrest = nNeighUp - MAX_SHARED;
-    uint32_t N4 = (Nrest / UNROLL_GLOBAL) * UNROLL_GLOBAL;
-    if constexpr (MAX_GLOBAL > 0) { // init the rest of the UNROLL_GLOBAL chunk of the global array
-      if (nNeighUp > MAX_SHARED && N4 < Nrest) {
-        N4 += UNROLL_GLOBAL;
+    const uint32_t nRest = nNeighUp - MAX_SHARED;
+    uint32_t nRestUnrolled = (nRest / UNROLL_GLOBAL) * UNROLL_GLOBAL;
+    if constexpr (MAX_GLOBAL > 1) { // init the rest of the UNROLL_GLOBAL chunk of the global array
+      if (nNeighUp > MAX_SHARED && nRestUnrolled < nRest) {
+        nRestUnrolled += UNROLL_GLOBAL;
         GPUCA_UNROLL(U(UNROLL_GLOBAL - 1), U(UNROLL_GLOBAL - 1))
         for (uint32_t k = 0; k + 1 < UNROLL_GLOBAL; k++) {
-          if (Nrest + k < N4) {
-            yzUp[2 * (Nrest + k)] = -1.e10f;
-            yzUp[2 * (Nrest + k) + 1] = -1.e10f;
-            neighUp[Nrest + k] = (calink)-1;
+          if (nRest + k < nRestUnrolled) {
+            yzUp[2 * (nRest + k)] = -1.e10f;
+            yzUp[2 * (nRest + k) + 1] = -1.e10f;
+            neighUp[nRest + k] = (calink)-1;
           }
         }
       }
@@ -229,8 +229,9 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh
         float zDnProjUp = s.mUpDx * (zDn - z);
 
         if constexpr (MAX_SHARED > 0) {
+          const uint32_t maxSharedUp = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_SHARED ? MAX_SHARED : CAMath::Min(nNeighUp, MAX_SHARED);
           GPUCA_UNROLL(U(MAX_SHARED), U(MAX_SHARED))
-          for (uint32_t iUp = 0; iUp < MAX_SHARED; iUp++) {
+          for (uint32_t iUp = 0; iUp < maxSharedUp; iUp++) {
             const float dy = yDnProjUp - s.mA1[iUp][iThread];
             const float dz = zDnProjUp - s.mA2[iUp][iThread];
             const float d = dy * dy + dz * dz;
@@ -244,7 +245,7 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh
 
         if constexpr (MAX_GLOBAL > 0) {
           if (nNeighUp > MAX_SHARED) {
-            for (uint32_t iUp = 0; iUp < N4; iUp += UNROLL_GLOBAL) {
+            for (uint32_t iUp = 0; iUp < nRestUnrolled; iUp += UNROLL_GLOBAL) {
               GPUCA_UNROLL(U(UNROLL_GLOBAL), U(UNROLL_GLOBAL))
               for (uint32_t k = 0; k < UNROLL_GLOBAL; k++) {
                 const uint32_t jUp = iUp + k;

GPU/GPUTracking/kernels.cmake

@@ -136,6 +136,8 @@ o2_gpu_add_kernel("GPUTrackingRefitKernel, mode0asGPU"                "= GLOBALR
 o2_gpu_add_kernel("GPUTrackingRefitKernel, mode1asTrackParCov"        "= GLOBALREFIT "                                        LB)
 
 o2_gpu_kernel_add_parameter(NEIGHBOURS_FINDER_MAX_NNEIGHUP
+                            NEIGHBOURS_FINDER_UNROLL_GLOBAL
+                            NEIGHBOURS_FINDER_UNROLL_SHARED
                             TRACKLET_SELECTOR_HITS_REG_SIZE
                             ALTERNATE_BORDER_SORT
                             SORT_BEFORE_FIT