GPU: Add possibility to dump raw data in case of error

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstruction.cxx

@@ -193,6 +193,7 @@ int32_t GPUReconstruction::Init()
     }
     mSlaves[i]->ClearAllocatedMemory();
   }
+  debugInit();
   return 0;
 }
 
@@ -469,6 +470,7 @@ int32_t GPUReconstruction::Exit()
   if (mInitialized) {
     ExitDevice();
   }
+  debugExit();
   mInitialized = false;
   return 0;
 }

GPU/GPUTracking/Base/GPUReconstruction.h

@@ -22,6 +22,7 @@
 #include <memory>
 #include <iosfwd>
 #include <vector>
+#include <functional>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -239,6 +240,7 @@ class GPUReconstruction
   virtual void PrintKernelOccupancies() {}
   double GetStatKernelTime() { return mStatKernelTime; }
   double GetStatWallTime() { return mStatWallTime; }
+  void setDebugDumpCallback(std::function<void()>&& callback = std::function<void()>(nullptr));
 
   // Threading
   std::shared_ptr<GPUReconstructionThreading> mThreading;
@@ -407,6 +409,13 @@ class GPUReconstruction
   };
   static std::shared_ptr<LibraryLoader> sLibCUDA, sLibHIP, sLibOCL;
 
+  // Debugging
+  struct debugInternal;
+  static std::unique_ptr<debugInternal> mDebugData;
+  bool mDebugEnabled = false;
+  void debugInit();
+  void debugExit();
+
   static GPUReconstruction* GPUReconstruction_Create_CPU(const GPUSettingsDeviceBackend& cfg);
 };
 

GPU/GPUTracking/Base/GPUReconstructionDebug.cxx

@@ -0,0 +1,103 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file GPUReconstructionDebug.cxx
+/// \author David Rohr
+
+#include "GPUReconstruction.h"
+#include "GPULogging.h"
+#include "GPUSettings.h"
+
+#include <signal.h>
+#include <functional>
+#include <unordered_map>
+#include <mutex>
+
+using namespace o2::gpu;
+
+struct GPUReconstruction::debugInternal {
+  std::function<void(int32_t, siginfo_t*, void*)> signalCallback;
+  std::function<void()> debugCallback = nullptr;
+  std::unordered_map<int32_t, struct sigaction> oldActions;
+  static void globalCallback(int32_t signal, siginfo_t* info, void* ucontext)
+  {
+    GPUReconstruction::mDebugData->signalCallback(signal, info, ucontext);
+  }
+};
+
+std::unique_ptr<GPUReconstruction::debugInternal> GPUReconstruction::mDebugData;
+
+void GPUReconstruction::debugInit()
+{
+  if (GetProcessingSettings().dumpOnFailure) {
+    static std::mutex initMutex;
+    {
+      std::lock_guard<std::mutex> guard(initMutex);
+      if (mDebugData) {
+        GPUFatal("Error handlers for debug dumps already set, cannot set them again");
+      }
+      mDebugData = std::make_unique<debugInternal>();
+    }
+    mDebugEnabled = true;
+    struct sigaction sa, oldsa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = GPUReconstruction::debugInternal::globalCallback;
+    sa.sa_flags = SA_SIGINFO;
+    int32_t sigs[] = {SIGINT, SIGABRT, SIGBUS, SIGTERM, SIGSEGV};
+    for (uint32_t i = 0; i < sizeof(sigs) / sizeof(sigs[0]); i++) {
+      if (sigaction(sigs[i], &sa, &oldsa)) {
+        GPUFatal("Error installing signal handler for error dump on signal %d", sigs[i]);
+      }
+      mDebugData->oldActions.emplace(sigs[i], oldsa);
+    }
+
+    mDebugData->signalCallback = [&oldActions = mDebugData->oldActions, myAction = std::move(sa)](int32_t signal, siginfo_t* info, void* ucontext) {
+      static std::mutex callbackMutex;
+      {
+        std::lock_guard<std::mutex> guard(callbackMutex);
+        GPUInfo("Received signal %d", signal);
+        if (mDebugData->debugCallback) {
+          GPUInfo("Running debug callback");
+          mDebugData->debugCallback();
+        }
+        mDebugData->debugCallback = nullptr;
+        sigaction(signal, &oldActions[signal], nullptr);
+        printf("DEF SIGNAL\n");
+        raise(signal);
+      }
+      sigaction(signal, &myAction, nullptr);
+    };
+  }
+}
+
+void GPUReconstruction::debugExit()
+{
+  if (!mDebugEnabled) {
+    return;
+  }
+  if (mDebugData) {
+    for (auto it = mDebugData->oldActions.begin(); it != mDebugData->oldActions.end(); it++) {
+      if (sigaction(it->first, &it->second, nullptr)) {
+        GPUFatal("Error restoring signal handler for signal %d", it->first);
+      }
+    }
+  }
+  mDebugEnabled = false;
+}
+
+void GPUReconstruction::setDebugDumpCallback(std::function<void()>&& callback)
+{
+  if (mMaster) {
+    mMaster->setDebugDumpCallback(std::move(callback));
+  } else if (mDebugEnabled && mDebugData) {
+    mDebugData->debugCallback = callback;
+  }
+}

GPU/GPUTracking/Base/GPUReconstructionLibrary.cxx

@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-/// \file GPUReconstruction.cxx
+/// \file GPUReconstructionLibrary.cxx
 /// \author David Rohr
 
 #ifdef _WIN32

GPU/GPUTracking/CMakeLists.txt

@@ -95,6 +95,7 @@ set(SRCS_NO_CINT
 set(SRCS_NO_H SectorTracker/GPUTPCTrackerDump.cxx
               Merger/GPUTPCGMMergerDump.cxx
               Base/GPUReconstructionLibrary.cxx
+              Base/GPUReconstructionDebug.cxx
               Global/GPUChainTrackingClusterizer.cxx
               Global/GPUChainTrackingTransformation.cxx
               Global/GPUChainTrackingTRD.cxx

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -360,6 +360,7 @@ AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from
 AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options")
 AddOption(printSettings, bool, false, "", 0, "Print all settings when initializing")
 AddOption(tpcFreeAllocatedMemoryAfterProcessing, bool, false, "", 0, "Clean all memory allocated by TPC when TPC processing done, only data written to external output resources will remain")
+AddOption(dumpOnFailure, bool, false, "", 0, "Dump raw data in case an error occured")
 AddVariable(eventDisplay, o2::gpu::GPUDisplayFrontendInterface*, nullptr)
 AddSubConfig(GPUSettingsProcessingRTC, rtc)
 AddSubConfig(GPUSettingsProcessingRTCtechnical, rtctech)

GPU/GPUTracking/Global/GPUChainTracking.cxx

@@ -705,10 +705,14 @@ int32_t GPUChainTracking::RunChain()
   }
   mRec->getGeneralStepTimer(GeneralStep::Prepare).Stop();
 
-  PrepareDebugOutput();
+  PrepareKernelDebugOutput();
 
   SynchronizeStream(0); // Synchronize all init copies that might be ongoing
 
+  if (GetProcessingSettings().dumpOnFailure) {
+    mRec->setDebugDumpCallback([]() { GPUInfo("CALLBACK"); });
+  }
+
   if (mIOPtrs.tpcCompressedClusters) {
     if (runRecoStep(RecoStep::TPCDecompression, &GPUChainTracking::RunTPCDecompression)) {
       return 1;
@@ -815,7 +819,7 @@ int32_t GPUChainTracking::RunChainFinalize()
     PrintOutputStat();
   }
 
-  PrintDebugOutput();
+  PrintKernelDebugOutput();
 
   // PrintMemoryRelations();
 

GPU/GPUTracking/Global/GPUChainTracking.h

@@ -231,8 +231,8 @@ class GPUChainTracking : public GPUChain
   int32_t DoProfile();
   void PrintMemoryRelations();
   void PrintMemoryStatistics() override;
-  void PrepareDebugOutput();
-  void PrintDebugOutput();
+  void PrepareKernelDebugOutput();
+  void PrintKernelDebugOutput();
   void PrintOutputStat();
   static void DumpClusters(std::ostream& out, const o2::tpc::ClusterNativeAccess* clusters);
   static void DebugSortCompressedClusters(o2::tpc::CompressedClustersFlat* cls);

GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx

@@ -185,7 +185,7 @@ void GPUChainTracking::PrintMemoryRelations()
   GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NOutputTrackClusters());
 }
 
-void GPUChainTracking::PrepareDebugOutput()
+void GPUChainTracking::PrepareKernelDebugOutput()
 {
 #ifdef GPUCA_KERNEL_DEBUGGER_OUTPUT
   const auto& threadContext = GetThreadContext();
@@ -198,7 +198,7 @@ void GPUChainTracking::PrepareDebugOutput()
 #endif
 }
 
-void GPUChainTracking::PrintDebugOutput()
+void GPUChainTracking::PrintKernelDebugOutput()
 {
 #ifdef GPUCA_KERNEL_DEBUGGER_OUTPUT
   const auto& threadContext = GetThreadContext();

GPU/GPUTracking/SectorTracker/GPUTPCStartHitsFinder.cxx

@@ -30,6 +30,7 @@ GPUdii() void GPUTPCStartHitsFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nThr
     } else {
       s.mNHits = -1;
     }
+    tracker.mData.mLinkUpData[10000000000] = 0;
   }
   GPUbarrier();
   GPUglobalref() const GPUTPCRow& GPUrestrict() row = tracker.mData.mRows[s.mIRow];