GPU: Add additional optional debbug dumps for validation

Author: davidrohr

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -295,7 +295,7 @@ AddOption(trdNCandidates, int32_t, 3, "", 0, "Number of branching track candidat
 AddOption(trdTrackModelO2, bool, false, "", 0, "Use O2 track model instead of GPU track model for TRD tracking")
 AddOption(debugLevel, int32_t, -1, "debug", 'd', "Set debug level (-2 = silent, -1 = autoselect (-2 for O2, 0 for standalone))")
 AddOption(allocDebugLevel, int32_t, 0, "allocDebug", 0, "Some debug output for memory allocations (without messing with normal debug level)")
-AddOption(debugMask, uint32_t, 262143, "", 0, "Mask for debug output dumps to file")
+AddOption(debugMask, uint32_t, (1 << 18) - 1, "debugMask", 0, "Mask for debug output dumps to file")
 AddOption(debugLogSuffix, std::string, "", "debugSuffix", 0, "Suffix for debug log files with --debug 6")
 AddOption(serializeGPU, int8_t, 0, "", 0, "Synchronize after each kernel call (bit 1) and DMA transfer (bit 2) and identify failures")
 AddOption(recoTaskTiming, bool, 0, "", 0, "Perform summary timing after whole reconstruction tasks")

GPU/GPUTracking/Global/GPUChainTrackingDebug.h

@@ -23,28 +23,29 @@ namespace o2::gpu
 {
 // NOTE: Values below 262144 are activated by default with --debug 6 in GPUSettingsList.h::debugMask
 enum GPUChainTrackingDebugFlags : uint32_t {
-  TPCSectorTrackingData = 1,
-  TPCPreLinks = 2,
-  TPCLinks = 4,
-  TPCStartHits = 8,
-  TPCTracklets = 16,
-  TPCSectorTracks = 32,
-  TPCHitWeights = 256,
-  TPCCompressedClusters = 512,
-  TPCDecompressedClusters = 1024,
-  TPCMergingRanges = 2048,
-  TPCMergingSectorTracks = 4096,
-  TPCMergingMergedTracks = 8192,
-  TPCMergingCollectedTracks = 16384,
-  TPCMergingCE = 32768,
-  TPCMergingRefit = 65536,
-  TPCClustererClusters = 131072,
-  TPCClusterer = 262144,
-  TPCClustererDigits = 262144 << 1,
-  TPCClustererPeaks = 262144 << 2,
-  TPCClustererSuppressedPeaks = 262144 << 3,
-  TPCClustererChargeMap = 262144 << 4,
-  TPCClustererZeroedCharges = 262144 << 5
+  TPCSectorTrackingData = 1 << 0,
+  TPCPreLinks = 1 << 1,
+  TPCLinks = 1 << 2,
+  TPCStartHits = 1 << 3,
+  TPCTracklets = 1 << 4,
+  TPCSectorTracks = 1 << 5,
+  TPCHitWeights = 1 << 6,
+  TPCMergingRanges = 1 << 7,
+  TPCMergingSectorTracks = 1 << 8,
+  TPCMergingMatching = 1 << 9,
+  TPCMergingCollectedTracks = 1 << 10,
+  TPCMergingCE = 1 << 11,
+  TPCMergingPrepareFit = 1 << 12,
+  TPCMergingRefit = 1 << 13,
+  TPCMergingLoopers = 1 << 14,
+  TPCCompressedClusters = 1 << 15,
+  TPCDecompressedClusters = 1 << 16,
+  TPCClustererClusters = 1 << 17,
+  TPCClustererDigits = 1 << 18,
+  TPCClustererPeaks = 1 << 19,
+  TPCClustererSuppressedPeaks = 1 << 20,
+  TPCClustererChargeMap = 1 << 21,
+  TPCClustererZeroedCharges = 1 << 22
 };
 
 template <class T, class S, typename... Args>

GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx

@@ -143,7 +143,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   runKernel<GPUTPCGMMergerMergeWithinPrepare>(GetGridAuto(0, deviceType));
   RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType);
   RunTPCTrackingMerger_Resolve(0, 1, deviceType);
-  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMergedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpMergedWithinSectors, *mDebugFile);
+  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedWithinSectors, *mDebugFile);
 
   runKernel<GPUTPCGMMergerClearLinks>(GetGridAuto(0, deviceType), false);
   runKernel<GPUMemClean16>({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter()));
@@ -158,7 +158,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   runKernel<GPUTPCGMMergerMergeSectorsPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 1);
   RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType);
   RunTPCTrackingMerger_Resolve(0, 1, deviceType);
-  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMergedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile);
+  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile);
 
   runKernel<GPUMemClean16>({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter()));
 
@@ -202,7 +202,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   runKernel<GPUTPCGMMergerPrepareClusters, 1>(GetGridAuto(0, deviceType));
   runKernel<GPUTPCGMMergerPrepareClusters, 2>(GetGridAuto(0, deviceType));
 
-  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, doGPU, Merger, &GPUTPCGMMerger::DumpFitPrepare, *mDebugFile);
+  DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingPrepareFit, doGPU, Merger, &GPUTPCGMMerger::DumpFitPrepare, *mDebugFile);
 
   if (doGPU) {
     CondWaitEvent(waitForTransfer, &mEvents->single);
@@ -240,6 +240,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
     }
     runKernel<GPUTPCGMMergerMergeLoopers, 1>(GetGridAuto(0, deviceType));
     runKernel<GPUTPCGMMergerMergeLoopers, 2>(doGPU ? GetGrid(Merger.Memory()->nLooperMatchCandidates, 0, deviceType) : GetGridAuto(0, deviceType));
+    DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingLoopers, Merger, &GPUTPCGMMerger::DumpLoopers, *mDebugFile);
   }
   DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, doGPU, Merger, &GPUTPCGMMerger::DumpFinal, *mDebugFile);
 

GPU/GPUTracking/Global/GPUChainTrackingSectorTracker.cxx

@@ -176,7 +176,9 @@ int32_t GPUChainTracking::RunTPCTrackingSectors_internal()
     }
 
     if (GetProcessingSettings().debugLevel >= 6) {
-      *mDebugFile << "\n\nReconstruction: Sector " << iSector << "/" << NSECTORS << std::endl;
+      if ((GetProcessingSettings().debugMask & 63)) {
+        *mDebugFile << "\n\nReconstruction: Sector " << iSector << "/" << NSECTORS << std::endl;
+      }
       if (GetProcessingSettings().debugMask & GPUChainTrackingDebugFlags::TPCSectorTrackingData) {
         if (doGPU) {
           TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &trk, -1, true);

GPU/GPUTracking/Merger/GPUTPCGMMergedTrack.h

@@ -95,6 +95,7 @@ class GPUTPCGMMergedTrack
   GPUd() void SetFlags(uint8_t v) { mFlags = v; }
   GPUd() void SetLegs(uint8_t v) { mLegs = v; }
   GPUd() uint8_t Legs() const { return mLegs; }
+  GPUd() uint8_t Flags() const { return mFlags; }
 
   GPUd() const gputpcgmmergertypes::GPUTPCOuterParam& OuterParam() const { return mOuterParam; }
   GPUd() gputpcgmmergertypes::GPUTPCOuterParam& OuterParam() { return mOuterParam; }

GPU/GPUTracking/Merger/GPUTPCGMMerger.h

@@ -201,6 +201,7 @@ class GPUTPCGMMerger : public GPUProcessor
   void DumpFitPrepare(std::ostream& out) const;
   void DumpRefit(std::ostream& out) const;
   void DumpFinal(std::ostream& out) const;
+  void DumpLoopers(std::ostream& out) const;
 
   template <int32_t mergeType>
   void MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, int32_t mergeMode, float weight, float frac) const;

GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx

@@ -204,11 +204,24 @@ void GPUTPCGMMerger::DumpRefit(std::ostream& out) const
     const auto& po = trk.OuterParam();
     out << "  Track " << i << ": OK " << trk.OK() << " Alpha " << trk.GetAlpha() << " X " << p.GetX() << " offset " << p.GetTZOffset() << " Y " << p.GetY() << " Z " << p.GetZ() << " SPhi " << p.GetSinPhi() << " Tgl " << p.GetDzDs() << " QPt " << p.GetQPt() << " NCl " << trk.NClusters() << " / " << trk.NClustersFitted() << " Cov " << p.GetErr2Y() << "/" << p.GetErr2Z()
         << " dEdx " << (trk.OK() && Param().dodEdxEnabled ? mOutputTracksdEdx[i].dEdxTotTPC : -1.f) << "/" << (trk.OK() && Param().dodEdxEnabled ? mOutputTracksdEdx[i].dEdxMaxTPC : -1.f)
-        << " Outer " << po.P[0] << "/" << po.P[1] << "/" << po.P[2] << "/" << po.P[3] << "/" << po.P[4] << "\n";
+        << " Outer " << po.P[0] << "/" << po.P[1] << "/" << po.P[2] << "/" << po.P[3] << "/" << po.P[4]
+        << " NFitted " << trk.NClustersFitted() << " legs " << (int)trk.Legs() << " flags " << (int)trk.Flags() << "\n";
   }
   out << std::setprecision(ss);
 }
 
+void GPUTPCGMMerger::DumpLoopers(std::ostream& out) const
+{
+  out << "\n TPC Merger Looper Afterburner\n";
+  for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) {
+    if (i && i % 100 == 0) {
+      out << "\n";
+    }
+    out << (int)mOutputTracks[i].MergedLooper() << " ";
+  }
+  out << "\n";
+}
+
 void GPUTPCGMMerger::DumpFinal(std::ostream& out) const
 {
   out << "\nTPC Merger Finalized\n";