GPU: Add memoryScaling fuzzing debug option

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstruction.cxx

@@ -304,7 +304,7 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice()
     mProcessingSettings->rtc.optConstexpr = false;
   }
 
-  mMemoryScalers->factor = GetProcessingSettings().memoryScalingFactor;
+  mMemoryScalers->scalingFactor = GetProcessingSettings().memoryScalingFactor;
   mMemoryScalers->conservative = GetProcessingSettings().conservativeMemoryEstimate;
   mMemoryScalers->returnMaxVal = GetProcessingSettings().forceMaxMemScalers != 0;
   if (GetProcessingSettings().forceMaxMemScalers > 1) {

GPU/GPUTracking/Base/GPUReconstructionCPU.cxx

@@ -216,6 +216,14 @@ int32_t GPUReconstructionCPU::ExitDevice()
 int32_t GPUReconstructionCPU::RunChains()
 {
   mMemoryScalers->temporaryFactor = 1.;
+  if (GetProcessingSettings().memoryScalingFuzz) {
+    static std::mt19937 rng;
+    static std::uniform_int_distribution<uint64_t> dist(0, 1000000);
+    uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz;
+    GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor);
+    mMemoryScalers->fuzzScalingFactor(fuzzFactor);
+  }
+
   mStatNEvents++;
   mNEventsProcessed++;
 

GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx

@@ -15,6 +15,8 @@
 #include "GPUMemorySizeScalers.h"
 #include "GPULogging.h"
 
+#include <random>
+
 using namespace o2::gpu;
 
 void GPUMemorySizeScalers::rescaleMaxMem(size_t newAvailableMemory)
@@ -36,3 +38,27 @@ void GPUMemorySizeScalers::rescaleMaxMem(size_t newAvailableMemory)
   tpcMaxMergedTrackHits = (double)tmp.tpcMaxMergedTrackHits * scaleFactor;
   availableMemory = newAvailableMemory;
 }
+
+double GPUMemorySizeScalers::getScalingFactor()
+{
+  if (!doFuzzing) {
+    return scalingFactor;
+  }
+  static std::uniform_int_distribution<uint32_t> dist(0, 1000000);
+  static std::mt19937 rng;
+  if (fuzzSeed) {
+    rng = std::mt19937(fuzzSeed);
+    fuzzLimit = dist(rng) / 10;
+    fuzzSeed = 0;
+  }
+  if (dist(rng) > fuzzLimit) {
+    return scalingFactor;
+  }
+  return scalingFactor * 0.000001 * dist(rng);
+}
+
+void GPUMemorySizeScalers::fuzzScalingFactor(uint64_t seed)
+{
+  fuzzSeed = seed;
+  doFuzzing = true;
+}

GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h

@@ -28,7 +28,9 @@ struct GPUMemorySizeScalers {
   size_t nITSTracks = 0;
 
   // General scaling factor
-  double factor = 1;
+  double scalingFactor = 1;
+  uint64_t fuzzSeed = 0;
+  uint64_t fuzzLimit = 0;
   double temporaryFactor = 1;
   bool conservative = 0;
 
@@ -64,11 +66,14 @@ struct GPUMemorySizeScalers {
   size_t tpcMaxMergedTrackHits = 200000000;
   size_t availableMemory = 20500000000;
   bool returnMaxVal = false;
+  bool doFuzzing = false;
 
   void rescaleMaxMem(size_t newAvailableMemory);
+  double getScalingFactor();
+  void fuzzScalingFactor(uint64_t seed);
   inline size_t getValue(size_t maxVal, size_t val)
   {
-    return returnMaxVal ? maxVal : (std::min<size_t>(maxVal, offset + val) * factor * temporaryFactor);
+    return returnMaxVal ? maxVal : (std::min<size_t>(maxVal, offset + val) * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor);
   }
 
   inline size_t NTPCPeaks(size_t tpcDigits, bool perSector = false) { return getValue(perSector ? tpcMaxPeaks : (GPUCA_NSECTORS * tpcMaxPeaks), hitOffset + tpcDigits * tpcPeaksPerDigit); }
@@ -81,7 +86,7 @@ struct GPUMemorySizeScalers {
   inline size_t NTPCSectorTrackHits(size_t tpcHits, uint8_t withRejection = 0) { return getValue(tpcMaxSectorTrackHits, tpcHits * (withRejection ? tpcSectorTrackHitsPerHitWithRejection : tpcSectorTrackHitsPerHit)); }
   inline size_t NTPCMergedTracks(size_t tpcSectorTracks) { return getValue(tpcMaxMergedTracks, tpcSectorTracks * (conservative ? 1.0 : tpcMergedTrackPerSectorTrack)); }
   inline size_t NTPCMergedTrackHits(size_t tpcSectorTrackHitss) { return getValue(tpcMaxMergedTrackHits, tpcSectorTrackHitss * tpcMergedTrackHitPerSectorHit); }
-  inline size_t NTPCUnattachedHitsBase1024(int32_t type) { return (returnMaxVal || conservative) ? 1024 : std::min<size_t>(1024, tpcCompressedUnattachedHitsBase1024[type] * factor * temporaryFactor); }
+  inline size_t NTPCUnattachedHitsBase1024(int32_t type) { return (returnMaxVal || conservative) ? 1024 : std::min<size_t>(1024, tpcCompressedUnattachedHitsBase1024[type] * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor); }
 };
 
 } // namespace o2::gpu

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -312,6 +312,7 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strageg
 AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory", min(0ul))
 AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul))
 AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers")
+AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1))
 AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conservative defaults for larger buffers during TPC processing")
 AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations")
 AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n")

GPU/GPUTracking/Interface/GPUO2Interface.cxx

@@ -110,7 +110,7 @@ int32_t GPUO2Interface::Initialize(const GPUO2InterfaceConfiguration& config)
       return (1);
     }
     if (!mCtx[i].mRec->IsGPU() && mCtx[i].mRec->GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
-      mCtx[i].mRec->MemoryScalers()->factor *= 2;
+      mCtx[i].mRec->MemoryScalers()->scalingFactor *= 2;
     }
   }
   if (mConfig->configProcessing.doublePipeline) {