GPU: Switch checkKernelFailures setting to more general serializeGPU

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstructionCPU.h

@@ -234,7 +234,7 @@ inline int32_t GPUReconstructionCPU::runKernel(krnlSetup&& setup, Args&&... args
   }
   double deviceTimerTime = 0.;
   int32_t retVal = runKernelImplWrapper(gpu_reconstruction_kernels::classArgument<S, I>(), cpuFallback, deviceTimerTime, std::forward<krnlSetup&&>(setup), std::forward<Args>(args)...);
-  if (GPUDebug(GetKernelName<S, I>(), stream, mProcessingSettings.checkKernelFailures)) {
+  if (GPUDebug(GetKernelName<S, I>(), stream, mProcessingSettings.serializeGPU & 1)) {
     throw std::runtime_error("kernel failure");
   }
   if (mProcessingSettings.debugLevel >= 1) {

GPU/GPUTracking/Base/GPUReconstructionDeviceBase.cxx

@@ -325,3 +325,17 @@ void GPUReconstructionDeviceBase::runConstantRegistrators()
     mDeviceConstantMemList.emplace_back(list[i]());
   }
 }
+
+size_t GPUReconstructionDeviceBase::TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst)
+{
+  if (!(res->Type() & GPUMemoryResource::MEMORY_GPU)) {
+    if (mProcessingSettings.debugLevel >= 4) {
+      GPUInfo("Skipped transfer of non-GPU memory resource: %s", res->Name());
+    }
+    return 0;
+  }
+  if (mProcessingSettings.debugLevel >= 3 && (strcmp(res->Name(), "ErrorCodes") || mProcessingSettings.debugLevel >= 4)) {
+    GPUInfo("Copying to %s: %s - %ld bytes", toGPU ? "GPU" : "Host", res->Name(), (int64_t)res->Size());
+  }
+  return GPUMemCpy(dst, src, res->Size(), stream, toGPU, ev, evList, nEvents);
+}

GPU/GPUTracking/Base/GPUReconstructionDeviceBase.h

@@ -56,7 +56,7 @@ class GPUReconstructionDeviceBase : public GPUReconstructionCPU
   virtual const GPUTPCTracker* CPUTracker(int32_t iSlice) { return &processors()->tpcTrackers[iSlice]; }
 
   int32_t GPUDebug(const char* state = "UNKNOWN", int32_t stream = -1, bool force = false) override = 0;
-  size_t TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) override = 0;
+  size_t TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) override;
   size_t GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override = 0;
   size_t GPUMemCpyAlways(bool onGpu, void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override;
   size_t WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream = -1, deviceEvent* ev = nullptr) override = 0;

GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu

@@ -519,21 +519,10 @@ size_t GPUReconstructionCUDA::GPUMemCpy(void* dst, const void* src, size_t size,
   if (ev) {
     GPUFailedMsg(cudaEventRecord(ev->get<cudaEvent_t>(), mInternals->Streams[stream == -1 ? 0 : stream]));
   }
-  return size;
-}
-
-size_t GPUReconstructionCUDA::TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst)
-{
-  if (!(res->Type() & GPUMemoryResource::MEMORY_GPU)) {
-    if (mProcessingSettings.debugLevel >= 4) {
-      GPUInfo("Skipped transfer of non-GPU memory resource: %s", res->Name());
-    }
-    return 0;
-  }
-  if (mProcessingSettings.debugLevel >= 3 && (strcmp(res->Name(), "ErrorCodes") || mProcessingSettings.debugLevel >= 4)) {
-    GPUInfo("Copying to %s: %s - %ld bytes", toGPU ? "GPU" : "Host", res->Name(), (int64_t)res->Size());
+  if (mProcessingSettings.serializeGPU & 2) {
+    GPUDebug(("GPUMemCpy " + std::to_string(toGPU)).c_str(), stream, true);
   }
-  return GPUMemCpy(dst, src, res->Size(), stream, toGPU, ev, evList, nEvents);
+  return size;
 }
 
 size_t GPUReconstructionCUDA::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev)
@@ -552,6 +541,9 @@ size_t GPUReconstructionCUDA::WriteToConstantMemory(size_t offset, const void* s
   if (ev && stream != -1) {
     GPUFailedMsg(cudaEventRecord(ev->get<cudaEvent_t>(), mInternals->Streams[stream]));
   }
+  if (mProcessingSettings.serializeGPU & 2) {
+    GPUDebug("WriteToConstantMemory", stream, true);
+  }
   return size;
 }
 
@@ -599,7 +591,7 @@ int32_t GPUReconstructionCUDA::GPUDebug(const char* state, int32_t stream, bool
   cudaError cuErr;
   cuErr = cudaGetLastError();
   if (cuErr != cudaSuccess) {
-    GPUError("CUDA Error %s while running kernel (%s) (Stream %d)", cudaGetErrorString(cuErr), state, stream);
+    GPUError("CUDA Error %s while running (%s) (Stream %d)", cudaGetErrorString(cuErr), state, stream);
     return (1);
   }
   if (!force && mProcessingSettings.debugLevel <= 0) {

GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.h

@@ -82,7 +82,6 @@ class GPUReconstructionCUDA : public GPUReconstructionKernels<GPUReconstructionC
   int32_t unregisterMemoryForGPU_internal(const void* ptr) override;
 
   size_t WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream = -1, deviceEvent* ev = nullptr) override;
-  size_t TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) override;
   size_t GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override;
   void ReleaseEvent(deviceEvent ev) override;
   void RecordMarker(deviceEvent ev, int32_t stream) override;

GPU/GPUTracking/Base/opencl-common/GPUReconstructionOCL.cxx

@@ -411,21 +411,10 @@ size_t GPUReconstructionOCL::GPUMemCpy(void* dst, const void* src, size_t size,
   } else {
     GPUFailedMsg(clEnqueueReadBuffer(mInternals->command_queue[stream == -1 ? 0 : stream], mInternals->mem_gpu, stream == -1, (char*)src - (char*)mDeviceMemoryBase, size, dst, nEvents, evList->getEventList<cl_event>(), ev->getEventList<cl_event>()));
   }
-  return size;
-}
-
-size_t GPUReconstructionOCL::TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst)
-{
-  if (!(res->Type() & GPUMemoryResource::MEMORY_GPU)) {
-    if (mProcessingSettings.debugLevel >= 4) {
-      GPUInfo("Skipped transfer of non-GPU memory resource: %s", res->Name());
-    }
-    return 0;
-  }
-  if (mProcessingSettings.debugLevel >= 3 && (strcmp(res->Name(), "ErrorCodes") || mProcessingSettings.debugLevel >= 4)) {
-    GPUInfo("Copying to %s: %s - %ld bytes", toGPU ? "GPU" : "Host", res->Name(), (int64_t)res->Size());
+  if (mProcessingSettings.serializeGPU & 2) {
+    GPUDebug(("GPUMemCpy " + std::to_string(toGPU)).c_str(), stream, true);
   }
-  return GPUMemCpy(dst, src, res->Size(), stream, toGPU, ev, evList, nEvents);
+  return size;
 }
 
 size_t GPUReconstructionOCL::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev)
@@ -434,6 +423,9 @@ size_t GPUReconstructionOCL::WriteToConstantMemory(size_t offset, const void* sr
     SynchronizeGPU();
   }
   GPUFailedMsg(clEnqueueWriteBuffer(mInternals->command_queue[stream == -1 ? 0 : stream], mInternals->mem_constant, stream == -1, offset, size, src, 0, nullptr, ev->getEventList<cl_event>()));
+  if (mProcessingSettings.serializeGPU & 2) {
+    GPUDebug("WriteToConstantMemory", stream, true);
+  }
   return size;
 }
 

GPU/GPUTracking/Base/opencl-common/GPUReconstructionOCL.h

@@ -50,7 +50,6 @@ class GPUReconstructionOCL : public GPUReconstructionDeviceBase
   bool IsEventDone(deviceEvent* evList, int32_t nEvents = 1) override;
 
   size_t WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream = -1, deviceEvent* ev = nullptr) override;
-  size_t TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) override;
   size_t GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override;
   void ReleaseEvent(deviceEvent ev) override;
   void RecordMarker(deviceEvent ev, int32_t stream) override;

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -227,7 +227,7 @@ AddOption(trdTrackModelO2, bool, false, "", 0, "Use O2 track model instead of GP
 AddOption(debugLevel, int32_t, -1, "debug", 'd', "Set debug level (-2 = silent, -1 = autoselect (-2 for O2, 0 for standalone))")
 AddOption(allocDebugLevel, int32_t, 0, "allocDebug", 0, "Some debug output for memory allocations (without messing with normal debug level)")
 AddOption(debugMask, int32_t, 262143, "", 0, "Mask for debug output dumps to file")
-AddOption(checkKernelFailures, bool, false, "", 0, "Synchronize after each kernel call and identify failing kernels")
+AddOption(serializeGPU, int8_t, 0, "", 0, "Synchronize after each kernel call (bit 1) and DMA transfer (bit 2) and identify failures")
 AddOption(deterministicGPUReconstruction, int32_t, -1, "", 0, "Make CPU and GPU debug output comparable (sort / skip concurrent parts), -1 = automatic if debugLevel >= 6")
 AddOption(showOutputStat, bool, false, "", 0, "Print some track output statistics")
 AddOption(runCompressionStatistics, bool, false, "compressionStat", 0, "Run statistics and verification for cluster compression")