GPU: Use aligned new/delete for some host allocations

Author: davidrohr

GPU/GPUTracking/Base/GPUReconstruction.cxx

@@ -716,8 +716,13 @@ void* GPUReconstruction::AllocateDirectMemory(size_t size, int32_t type)
     throw std::runtime_error("Requested invalid memory typo for unmanaged allocation");
   }
   if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
-    mUnmanagedChunks.emplace_back(new char[size + GPUCA_BUFFER_ALIGNMENT]);
-    return GPUProcessor::alignPointer<GPUCA_BUFFER_ALIGNMENT>(mUnmanagedChunks.back().get());
+    char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
+    if ((type & GPUMemoryResource::MEMORY_STACK)) {
+      mNonPersistentIndividualDirectAllocations.emplace_back(retVal, alignedDeleter());
+    } else {
+      mDirectMemoryChunks.emplace_back(retVal, alignedDeleter());
+    }
+    return retVal;
   } else {
     if (mVolatileMemoryStart && !mDeviceMemoryAsVolatile && (type & GPUMemoryResource::MEMORY_GPU) && !(type & GPUMemoryResource::MEMORY_STACK)) {
       GPUError("Must not allocate direct memory while volatile chunks are allocated");
@@ -765,8 +770,9 @@ void* GPUReconstruction::AllocateVolatileMemory(size_t size, bool device)
   if (device) {
     return AllocateVolatileDeviceMemory(size);
   }
-  mVolatileChunks.emplace_back(new char[size + GPUCA_BUFFER_ALIGNMENT]);
-  return GPUProcessor::alignPointer<GPUCA_BUFFER_ALIGNMENT>(mVolatileChunks.back().get());
+  char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
+  mVolatileChunks.emplace_back(retVal, alignedDeleter());
+  return retVal;
 }
 
 void GPUReconstruction::MakeFutureDeviceMemoryAllocationsVolatile()
@@ -851,7 +857,7 @@ void GPUReconstruction::FreeRegisteredMemory(GPUMemoryResource* res)
 
 void GPUReconstruction::PushNonPersistentMemory(uint64_t tag)
 {
-  mNonPersistentMemoryStack.emplace_back(mHostMemoryPoolEnd, mDeviceMemoryPoolEnd, mNonPersistentIndividualAllocations.size(), tag);
+  mNonPersistentMemoryStack.emplace_back(mHostMemoryPoolEnd, mDeviceMemoryPoolEnd, mNonPersistentIndividualAllocations.size(), mNonPersistentIndividualDirectAllocations.size(), tag);
 }
 
 void GPUReconstruction::PopNonPersistentMemory(RecoStep step, uint64_t tag)
@@ -862,11 +868,11 @@ void GPUReconstruction::PopNonPersistentMemory(RecoStep step, uint64_t tag)
   if (mNonPersistentMemoryStack.size() == 0) {
     GPUFatal("Trying to pop memory state from empty stack");
   }
-  if (tag != 0 && std::get<3>(mNonPersistentMemoryStack.back()) != tag) {
-    GPUFatal("Tag mismatch when popping non persistent memory from stack : pop %s vs on stack %s", qTag2Str(tag).c_str(), qTag2Str(std::get<3>(mNonPersistentMemoryStack.back())).c_str());
+  if (tag != 0 && std::get<4>(mNonPersistentMemoryStack.back()) != tag) {
+    GPUFatal("Tag mismatch when popping non persistent memory from stack : pop %s vs on stack %s", qTag2Str(tag).c_str(), qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str());
   }
   if ((GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) && (IsGPU() || GetProcessingSettings().forceHostMemoryPoolSize)) {
-    printf("Allocated memory after %30s (%8s) (Stack %zu): ", GPUDataTypes::RECO_STEP_NAMES[getRecoStepNum(step, true)], qTag2Str(std::get<3>(mNonPersistentMemoryStack.back())).c_str(), mNonPersistentMemoryStack.size());
+    printf("Allocated memory after %30s (%8s) (Stack %zu): ", GPUDataTypes::RECO_STEP_NAMES[getRecoStepNum(step, true)], qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str(), mNonPersistentMemoryStack.size());
     PrintMemoryOverview();
     printf("%76s", "");
     PrintMemoryMax();
@@ -882,6 +888,7 @@ void GPUReconstruction::PopNonPersistentMemory(RecoStep step, uint64_t tag)
     res->mPtrDevice = nullptr;
   }
   mNonPersistentIndividualAllocations.resize(std::get<2>(mNonPersistentMemoryStack.back()));
+  mNonPersistentIndividualDirectAllocations.resize(std::get<3>(mNonPersistentMemoryStack.back()));
   mNonPersistentMemoryStack.pop_back();
 }
 
@@ -917,9 +924,11 @@ void GPUReconstruction::ClearAllocatedMemory(bool clearOutputs)
       FreeRegisteredMemory(i);
     }
   }
-  mUnmanagedChunks.clear();
   mNonPersistentMemoryStack.clear();
   mNonPersistentIndividualAllocations.clear();
+  mDirectMemoryChunks.clear();
+  mNonPersistentIndividualDirectAllocations.clear();
+  mVolatileChunks.clear();
   mVolatileMemoryStart = nullptr;
   if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
     mHostMemoryPool = GPUProcessor::alignPointer<GPUCA_MEMALIGN>(mHostMemoryPermanent);

GPU/GPUTracking/Base/GPUReconstruction.h

@@ -69,8 +69,6 @@ class GPUReconstruction
   class LibraryLoader; // These must be the first members to ensure correct destructor order!
   std::shared_ptr<LibraryLoader> mMyLib = nullptr;
   std::vector<GPUMemoryResource> mMemoryResources;
-  std::vector<std::unique_ptr<char[]>> mUnmanagedChunks;
-  std::vector<std::unique_ptr<char[]>> mVolatileChunks;
   std::vector<std::unique_ptr<GPUChain>> mChains;
 
  public:
@@ -373,9 +371,15 @@ class GPUReconstruction
     GPUProcessor* proc = nullptr;
     std::vector<uint16_t> res;
   };
+  struct alignedDeleter {
+    void operator()(void* ptr) { ::operator delete(ptr, std::align_val_t(GPUCA_BUFFER_ALIGNMENT)); };
+  };
   std::unordered_map<GPUMemoryReuse::ID, MemoryReuseMeta> mMemoryReuse1to1;
-  std::vector<std::tuple<void*, void*, size_t, uint64_t>> mNonPersistentMemoryStack;
+  std::vector<std::tuple<void*, void*, size_t, size_t, uint64_t>> mNonPersistentMemoryStack; // hostPoolAddress, devicePoolAddress, individualAllocationCount, directIndividualAllocationCound, tag
   std::vector<GPUMemoryResource*> mNonPersistentIndividualAllocations;
+  std::vector<std::unique_ptr<char[], alignedDeleter>> mNonPersistentIndividualDirectAllocations;
+  std::vector<std::unique_ptr<char[], alignedDeleter>> mDirectMemoryChunks;
+  std::vector<std::unique_ptr<char[], alignedDeleter>> mVolatileChunks;
 
   std::unique_ptr<GPUReconstructionPipelineContext> mPipelineContext;