GPU TPC: Implement ambiguity solving for rebuild tracks cluster association

Author: davidrohr

GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h

@@ -64,6 +64,11 @@
     #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256
     #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 128, 1
     #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 2
     #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 128, 2
@@ -193,6 +198,11 @@
     #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256
     #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 64, 2
     #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 3
     #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 32, 1
@@ -255,6 +265,11 @@
     #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256
     #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256
     #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 128
     #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 2
     #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 32, 1
@@ -425,6 +440,21 @@
   #ifndef GPUCA_LB_GPUTPCGMMergerFinalize_step2
     #define GPUCA_LB_GPUTPCGMMergerFinalize_step2 256
   #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_prepare
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_compute
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared
+    #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256
+  #endif
   #ifndef GPUCA_LB_GPUTPCGMMergerMergeLoopers_step0
     #define GPUCA_LB_GPUTPCGMMergerMergeLoopers_step0 256
   #endif

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -140,12 +140,13 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2
 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)")
 AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions")
 AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track")
+AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks")
 AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits")
 AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit")
 AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128")
 AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128")
 AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)")
-AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following, 8: mirroring)")
+AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)")
 AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)")
 AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging")
 AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)")

GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx

@@ -187,8 +187,8 @@ void GPUChainTracking::PrintMemoryRelations()
     GPUInfo("MEMREL SectorTrackHits NCl %d NTrkH %d", processors()->tpcTrackers[i].NHitsTotal(), *processors()->tpcTrackers[i].NTrackHits());
   }
   if (processors()->tpcMerger.Memory()) {
-    GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTracks());
-    GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTrackClusters());
+    GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTracks());
+    GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTrackClusters());
   }
 }
 
@@ -217,7 +217,7 @@ void GPUChainTracking::PrintKernelDebugOutput()
 void GPUChainTracking::PrintOutputStat()
 {
   int32_t nTracks = 0, nAttachedClusters = 0, nAttachedClustersFitted = 0, nAdjacentClusters = 0;
-  uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NMaxClusters();
+  uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NClusters();
   if (GetProcessingSettings().createO2Output > 1) {
     nTracks = mIOPtrs.nOutputTracksTPCO2;
     nAttachedClusters = mIOPtrs.nMergedTrackHits;

GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx

@@ -195,12 +195,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
     Merger.CheckCollectedTracks();
   }
 
-  uint32_t maxId = Merger.NMaxClusters();
-  if (maxId > Merger.NMaxClusters()) {
-    throw std::runtime_error("mNMaxClusters too small");
-  }
-  runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount()));
-  runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment()));
+  runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount()));
+  runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment()));
   runKernel<GPUTPCGMMergerPrepareForFit, 0>(GetGridAuto(0, deviceType));
   CondWaitEvent(waitForTransfer, &mEvents->single);
   runKernel<GPUTPCGMMergerSortTracksQPt>(GetGridAuto(0, deviceType));
@@ -230,6 +226,16 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   }
   runKernel<GPUTPCGMMergerTrackFit>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0);
   if (param().rec.tpc.rebuildTrackInFit) {
+    runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights()));
+    runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::prepare>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0);
+    for (int32_t i = 0; i < param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) {
+      runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::compute>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i);
+      if (i + 1 < param().rec.tpc.rebuildTrackInFitClusterCandidates) {
+        runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::resolve1>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i);
+      }
+    }
+    runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::resolve2>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0);
+    runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::resolveShared>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0);
     runKernel<GPUTPCGMMergerTrackFit>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1);
   }
   runKernel<GPUTPCGMMergerFollowLoopers>(GetGridAuto(0));
@@ -268,7 +274,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
         GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0);
       }
       GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0);
-      GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0);
+      GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0);
     }
     if (GetProcessingSettings().outputSharedClusterMap) {
       TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent);
@@ -358,7 +364,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   }
 
   if (GetProcessingSettings().debugLevel >= 2) {
-    GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NClusters());
+    GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NSectorHits());
   }
   return 0;
 }

GPU/GPUTracking/Global/GPUErrorCodes.h

@@ -33,16 +33,16 @@ GPUCA_ERROR_CODE(13, ERROR_SECTORDATA_HITINROW_OVERFLOW, SectorRow, Value, Max)
 GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max)
 GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value)
 GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max)
-GPUCA_ERROR_CODE(17, ERROR_MERGER_TRACK_OVERFLOW, Value, Max)
-GPUCA_ERROR_CODE(18, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max)
-GPUCA_ERROR_CODE(19, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max)
-GPUCA_ERROR_CODE(20, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max)
-GPUCA_ERROR_CODE(21, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max)
-GPUCA_ERROR_CODE(22, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max)
-GPUCA_ERROR_CODE(23, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max)
-GPUCA_ERROR_CODE(24, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max)
-GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER
-
+GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max)
+GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max)
+GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max)
+GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max)
+GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max)
+GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max)
+GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max)
+GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max)
+GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max)
+GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER)                                           // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER
 GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow)                                  // Data from invalid row is skipped
 GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten)  // Invalid number of ADC samples in header, existing samples were decoded
 GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete