GPU: Add GPUCommonAlignedAlloc.h for aligned buffers, and use it for TPCFastTransformPOD

Author: davidrohr

Detectors/TPC/calibration/include/TPCCalibration/CalculatedEdx.h

@@ -228,7 +228,7 @@ class CalculatedEdx
   std::vector<TPCClRefElem>* mTPCTrackClIdxVecInput{nullptr}; ///< input vector with TPC tracks cluster indicies
   const o2::tpc::ClusterNativeAccess* mClusterIndex{nullptr}; ///< needed to access clusternative with tpctracks
   const o2::gpu::TPCFastTransformPOD* mTPCCorrMap{nullptr};   ///< cluster correction maps helper
-  std::vector<char> mTPCCorrMapBuffer;
+  o2::gpu::aligned_unique_buffer_ptr<o2::gpu::TPCFastTransformPOD> mTPCCorrMapBuffer;
   std::vector<unsigned char> mTPCRefitterShMap;                  ///< externally set TPC clusters sharing map
   std::vector<unsigned int> mTPCRefitterOccMap;                  ///< externally set TPC clusters occupancy map
   std::unique_ptr<o2::gpu::GPUO2InterfaceRefit> mRefit{nullptr}; ///< TPC refitter used for TPC tracks refit during the reconstruction
@@ -247,4 +247,4 @@ class CalculatedEdx
 
 } // namespace o2::tpc
 
-#endif
+#endif

Detectors/TPC/calibration/include/TPCCalibration/TrackDump.h

@@ -77,8 +77,8 @@ class TrackDump
     float gyc(float vertexTime = 0) const;
     float zc(float vertexTime = 0) const;
 
-    inline static std::vector<char> corrMapBuffer;                      // buffer for owning the correction map in case of update during runtime
-    inline static const o2::gpu::TPCFastTransformPOD* corrMap{nullptr}; // local copy of the correction map for quick access to the transform functions
+    inline static o2::gpu::aligned_unique_buffer_ptr<o2::gpu::TPCFastTransformPOD> corrMapBuffer; // buffer for owning the correction map in case of update during runtime
+    inline static const o2::gpu::TPCFastTransformPOD* corrMap{nullptr};                           // local copy of the correction map for quick access to the transform functions
     static void loadCorrMaps(std::string_view corrMapFile, std::string_view corrMapFileRef = "");
     ClassDefNV(ClusterNativeAdd, 1);
   };

Detectors/TPC/calibration/src/CalculatedEdx.cxx

@@ -32,10 +32,10 @@ using namespace o2::tpc;
 
 CalculatedEdx::CalculatedEdx()
 {
-  std::vector<char> buffer;
+  gpu::aligned_unique_buffer_ptr<gpu::TPCFastTransformPOD> buffer;
   gpu::TPCFastTransformPOD::create(buffer, *TPCFastTransformHelperO2::instance()->create(0));
   mTPCCorrMapBuffer = std::move(buffer);
-  mTPCCorrMap = &gpu::TPCFastTransformPOD::get(mTPCCorrMapBuffer.data());
+  mTPCCorrMap = mTPCCorrMapBuffer.get();
 }
 
 void CalculatedEdx::setMembers(std::vector<o2::tpc::TPCClRefElem>* tpcTrackClIdxVecInput, const o2::tpc::ClusterNativeAccess& clIndex, std::vector<o2::tpc::TrackTPC>* vTPCTracksArrayInp)

Detectors/TPC/calibration/src/TrackDump.cxx

@@ -236,8 +236,8 @@ float TrackDump::ClusterNativeAdd::zc(float vertexTime) const
 void TrackDump::ClusterNativeAdd::loadCorrMaps(std::string_view corrMapFile, std::string_view corrMapFileRef)
 {
   auto fastTransformTmp = gpu::TPCFastTransform::loadFromFile(corrMapFile.data());
-  std::vector<char> buffer;
+  o2::gpu::aligned_unique_buffer_ptr<o2::gpu::TPCFastTransformPOD> buffer;
   gpu::TPCFastTransformPOD::create(buffer, *fastTransformTmp);
   corrMapBuffer = std::move(buffer);
-  corrMap = &gpu::TPCFastTransformPOD::get(corrMapBuffer.data());
+  corrMap = corrMapBuffer.get();
 }

Detectors/TPC/reconstruction/test/testGPUCATracking.cxx

@@ -74,9 +74,9 @@ BOOST_AUTO_TEST_CASE(CATracking_test1)
   config.configWorkflow.outputs.set(gpudatatypes::InOutType::TPCMergedTracks);
 
   auto fastTransformTmp = TPCFastTransformHelperO2::instance()->create(0);
-  std::vector<char> fastTransformBuf;
+  aligned_unique_buffer_ptr<TPCFastTransformPOD> fastTransformBuf;
   TPCFastTransformPOD::create(fastTransformBuf, *fastTransformTmp);
-  config.configCalib.fastTransform = &TPCFastTransformPOD::get(fastTransformBuf.data());
+  config.configCalib.fastTransform = fastTransformBuf.get();
 
   auto dEdxCalibContainer = GPUO2InterfaceUtils::getCalibdEdxContainerDefault();
   config.configCalib.dEdxCalibContainer = dEdxCalibContainer.get();

Detectors/TPC/workflow/src/TPCScalerSpec.cxx

@@ -219,7 +219,8 @@ class TPCScalerSpec : public Task
 
     Output corrMapOutput{header::gDataOriginTPC, "TPCCORRMAP", 0};
     auto outputBuffer = o2::pmr::vector<char>(pc.outputs().getMemoryResource(corrMapOutput));
-    auto* pod = TPCFastTransformPOD::create(outputBuffer, finalMap.getCorrection());
+    outputBuffer.resize(TPCFastTransformPOD::estimateSize(finalMap.getCorrection()));
+    auto* pod = TPCFastTransformPOD::create(outputBuffer.data(), outputBuffer.size(), finalMap.getCorrection());
     const auto& vd = mTPCVDriftHelper.getVDriftObject();
     o2::tpc::TPCFastTransformHelperO2::instance()->updateCalibration(*pod, 0, vd.corrFact, vd.refVDrift, vd.getTimeOffset());
     pc.outputs().adoptContainer(corrMapOutput, std::move(outputBuffer));

GPU/Common/CMakeLists.txt

@@ -13,6 +13,7 @@ set(MODULE GPUCommon)
 
 set(HDRS_INSTALL
     GPUCommonAlgorithm.h
+    GPUCommonAlignedAlloc.h
     GPUCommonDef.h
     GPUCommonDefAPI.h
     GPUCommonHelpers.h

GPU/Common/GPUCommonAlignedAlloc.h

@@ -0,0 +1,61 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file GPUCommonAlignedAlloc.h
+/// \author David Rohr
+
+#ifndef GPUCOMMONAKUGBEDALLOC_H
+#define GPUCOMMONAKUGBEDALLOC_H
+
+#include <memory>
+
+namespace o2::gpu
+{
+
+template <typename T, std::size_t MIN_ALIGN = 0>
+struct alignedDeleter {
+  void operator()(void* ptr) { ::operator delete(ptr, std::align_val_t(std::max(MIN_ALIGN, alignof(T)))); };
+};
+
+template <typename T, std::size_t MIN_ALIGN = 0>
+struct alignedAllocator {
+  using value_type = T;
+  T* allocate(std::size_t n)
+  {
+    return (T*)::operator new(n, std::align_val_t(std::max(MIN_ALIGN, alignof(T))));
+  }
+  void deallocate(T* ptr, std::size_t)
+  {
+    alignedDeleter<T, MIN_ALIGN>()(ptr);
+  }
+};
+
+template <typename T>
+struct aligned_unique_buffer_ptr : public std::unique_ptr<char[], alignedDeleter<T>> {
+  aligned_unique_buffer_ptr() = default;
+  aligned_unique_buffer_ptr(size_t n) { alloc(n); }
+  aligned_unique_buffer_ptr(T* ptr) { std::unique_ptr<char[], alignedDeleter<T>>::reset((char*)ptr); }
+  char* getraw() { return std::unique_ptr<char[], alignedDeleter<T>>::get(); }
+  const char* getraw() const { return std::unique_ptr<char[], alignedDeleter<T>>::get(); }
+  T* get() { return (T*)std::unique_ptr<char[], alignedDeleter<T>>::get(); }
+  const T* get() const { return (T*)std::unique_ptr<char[], alignedDeleter<T>>::get(); }
+  T* operator->() { return get(); }
+  const T* operator->() const { return get(); }
+  T* alloc(std::size_t n)
+  {
+    std::unique_ptr<char[], alignedDeleter<T>>::reset((char*)alignedAllocator<T>().allocate(n));
+    return get();
+  }
+};
+
+} // namespace o2::gpu
+
+#endif // GPUCOMMONAKUGBEDALLOC_H

GPU/GPUTracking/Base/GPUReconstruction.cxx

@@ -467,7 +467,7 @@ int32_t GPUReconstruction::Exit()
       if (mMemoryResources[i].mReuse >= 0) {
         continue;
       }
-      operator delete(mMemoryResources[i].mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
+      ::operator delete(mMemoryResources[i].mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
       mMemoryResources[i].mPtr = mMemoryResources[i].mPtrDevice = nullptr;
     }
   }
@@ -630,7 +630,7 @@ void GPUReconstruction::AllocateRegisteredMemoryInternal(GPUMemoryResource* res,
   if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && (control == nullptr || control->useInternal())) {
     if (!(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) {
       if (res->mPtrDevice && res->mReuse < 0) {
-        operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
+        ::operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
       }
       res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
       if (res->mReuse >= 0) {
@@ -640,7 +640,7 @@ void GPUReconstruction::AllocateRegisteredMemoryInternal(GPUMemoryResource* res,
         }
         res->mPtrDevice = mMemoryResources[res->mReuse].mPtrDevice;
       } else {
-        res->mPtrDevice = operator new(res->mSize + GPUCA_BUFFER_ALIGNMENT, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
+        res->mPtrDevice = ::operator new(res->mSize + GPUCA_BUFFER_ALIGNMENT, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
       }
       res->mPtr = GPUProcessor::alignPointer<GPUCA_BUFFER_ALIGNMENT>(res->mPtrDevice);
       res->SetPointers(res->mPtr);
@@ -733,9 +733,9 @@ void* GPUReconstruction::AllocateDirectMemory(size_t size, int32_t type)
   if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
     char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
     if ((type & GPUMemoryResource::MEMORY_STACK)) {
-      mNonPersistentIndividualDirectAllocations.emplace_back(retVal, alignedDeleter());
+      mNonPersistentIndividualDirectAllocations.emplace_back(retVal, alignedDefaultBufferDeleter());
     } else {
-      mDirectMemoryChunks.emplace_back(retVal, alignedDeleter());
+      mDirectMemoryChunks.emplace_back(retVal, alignedDefaultBufferDeleter());
     }
     return retVal;
   }
@@ -798,7 +798,7 @@ void* GPUReconstruction::AllocateVolatileMemory(size_t size, bool device)
   }
   char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
   stdspinlock spinlock(mMemoryMutex);
-  mVolatileChunks.emplace_back(retVal, alignedDeleter());
+  mVolatileChunks.emplace_back(retVal, alignedDefaultBufferDeleter());
   return retVal;
 }
 
@@ -876,7 +876,7 @@ void GPUReconstruction::FreeRegisteredMemory(GPUMemoryResource* res)
     std::cout << "Freeing " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
   }
   if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && res->mReuse < 0) {
-    operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
+    ::operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
   }
   res->mPtr = nullptr;
   res->mPtrDevice = nullptr;
@@ -916,7 +916,7 @@ void GPUReconstruction::PopNonPersistentMemory(RecoStep step, uint64_t tag, cons
       std::cout << "Freeing NonPersistent " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
     }
     if (res->mReuse < 0) {
-      operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
+      ::operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
     }
     res->mPtr = nullptr;
     res->mPtrDevice = nullptr;

GPU/GPUTracking/Base/GPUReconstruction.h

@@ -32,6 +32,7 @@
 #include "GPUOutputControl.h"
 #include "GPUParam.h"
 #include "GPUConstantMem.h"
+#include "GPUCommonAlignedAlloc.h"
 #include "GPUDef.h"
 
 namespace o2::its
@@ -381,15 +382,13 @@ class GPUReconstruction
     GPUProcessor* proc = nullptr;
     std::vector<uint16_t> res;
   };
-  struct alignedDeleter {
-    void operator()(void* ptr) { ::operator delete[](ptr, std::align_val_t(GPUCA_BUFFER_ALIGNMENT)); };
-  };
   std::unordered_map<GPUMemoryReuse::ID, MemoryReuseMeta> mMemoryReuse1to1;
   std::vector<std::tuple<void*, void*, size_t, size_t, uint64_t>> mNonPersistentMemoryStack; // hostPoolAddress, devicePoolAddress, individualAllocationCount, directIndividualAllocationCound, tag
   std::vector<GPUMemoryResource*> mNonPersistentIndividualAllocations;
-  std::vector<std::unique_ptr<char[], alignedDeleter>> mNonPersistentIndividualDirectAllocations;
-  std::vector<std::unique_ptr<char[], alignedDeleter>> mDirectMemoryChunks;
-  std::vector<std::unique_ptr<char[], alignedDeleter>> mVolatileChunks;
+  using alignedDefaultBufferDeleter = alignedDeleter<char, GPUCA_BUFFER_ALIGNMENT>;
+  std::vector<std::unique_ptr<char[], alignedDefaultBufferDeleter>> mNonPersistentIndividualDirectAllocations;
+  std::vector<std::unique_ptr<char[], alignedDefaultBufferDeleter>> mDirectMemoryChunks;
+  std::vector<std::unique_ptr<char[], alignedDefaultBufferDeleter>> mVolatileChunks;
   std::atomic_flag mMemoryMutex = ATOMIC_FLAG_INIT;
 
   std::unique_ptr<GPUReconstructionPipelineContext> mPipelineContext;