GPU ITS tracking: make tracking mode available in configKeyVals + fixes (#13056)

* Fix usage of VertexerTraits from within the GPU reco WF

* Prepare settings and cleanup interface

* Temporarily re-enable the gpu-wf-spec in itsreco-wf

* Extend configuration

Author: mconcas

Detectors/ITSMFT/ITS/tracking/GPU/cuda/VertexerTraitsGPU.cu

@@ -93,7 +93,6 @@ VertexerTraitsGPU::VertexerTraitsGPU()
 
 VertexerTraitsGPU::~VertexerTraitsGPU()
 {
-  gpu::utils::gpuFree(mDeviceIndexTableUtils);
 }
 
 void VertexerTraitsGPU::initialise(const TrackingParameters& trackingParams)

Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h

@@ -34,7 +34,8 @@ namespace its
 enum class TrackingMode {
   Sync,
   Async,
-  Cosmics
+  Cosmics,
+  Unset, // Special value to leave a default in case we want to override via Configurable Params
 };
 
 std::string asString(TrackingMode mode);
@@ -146,6 +147,7 @@ struct TimeFrameGPUParameters {
   size_t maxVerticesCapacity = 5e4;
   size_t nMaxROFs = 1e3;
   size_t nTimeFrameChunks = 3;
+  size_t nROFsPerChunk = 768; // pp defaults
   int maxGPUMemoryGB = -1;
 };
 

Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h

@@ -78,6 +78,8 @@ struct TrackerParamConfig : public o2::conf::ConfigurableParamHelper<TrackerPara
   int nROFsPerIterations = 0;
   bool perPrimaryVertexProcessing = false;
   bool saveTimeBenchmarks = false;
+  bool overrideBeamEstimation = false; // used by gpuwf only
+  int trackingMode = -1;               // -1: unset, 0=sync, 1=async, 2=cosmics used by gpuwf only
 
   O2ParamDef(TrackerParamConfig, "ITSCATrackerParam");
 };

Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h

@@ -34,20 +34,21 @@ class ITSTrackingInterface
  public:
   ITSTrackingInterface(bool isMC,
                        int trgType,
-                       const TrackingMode trMode,
                        const bool overrBeamEst)
     : mIsMC{isMC},
       mUseTriggers{trgType},
-      mMode{trMode},
       mOverrideBeamEstimation{overrBeamEst}
   {
   }
 
   void setClusterDictionary(const o2::itsmft::TopologyDictionary* d) { mDict = d; }
   void setMeanVertex(const o2::dataformats::MeanVertexObject* v)
   {
-    if (!v) {
+    if (v == nullptr) {
+      LOGP(error, "Mean Vertex Object is nullptr");
       return;
+    } else {
+      LOGP(info, "Mean Vertex set with x: {} y: {}", v->getX(), v->getY());
     }
     mMeanVertex = v;
   }
@@ -61,13 +62,20 @@ class ITSTrackingInterface
 
   // Custom
   void setTraitsFromProvider(VertexerTraits*, TrackerTraits*, TimeFrame*);
+  void setTrackingMode(TrackingMode mode = TrackingMode::Unset)
+  {
+    if (mode == TrackingMode::Unset) {
+      LOGP(fatal, "ITS Tracking mode Unset is meant to be a default. Specify the mode");
+    }
+    mMode = mode;
+  }
 
  private:
   bool mIsMC = false;
   bool mRunVertexer = true;
   bool mCosmicsProcessing = false;
   int mUseTriggers = 0;
-  TrackingMode mMode = TrackingMode::Sync;
+  TrackingMode mMode = TrackingMode::Unset;
   bool mOverrideBeamEstimation = false;
   const o2::itsmft::TopologyDictionary* mDict = nullptr;
   std::unique_ptr<Tracker> mTracker = nullptr;

Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h

@@ -85,13 +85,13 @@ class VertexerTraits
                             TimeFrame*,
                             std::vector<o2::MCCompLabel>*);
 
-  VertexingParameters getVertexingParameters() const { return mVrtParams; }
   static const std::vector<std::pair<int, int>> selectClusters(const int* indexTable,
                                                                const std::array<int, 4>& selectedBinsRect,
                                                                const IndexTableUtils& utils);
 
   // utils
   VertexingParameters& getVertexingParameters() { return mVrtParams; }
+  VertexingParameters getVertexingParameters() const { return mVrtParams; }
   void setIsGPU(const unsigned char isgpu) { mIsGPU = isgpu; };
   unsigned char getIsGPU() const { return mIsGPU; };
   void dumpVertexerTraits();

Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx

@@ -22,6 +22,8 @@ std::string asString(TrackingMode mode)
       return "async";
     case TrackingMode::Cosmics:
       return "cosmics";
+    case TrackingMode::Unset:
+      return "unset";
   }
   return "unknown";
 }

Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx

@@ -455,6 +455,7 @@ void Tracker::rectifyClusterIndices()
 void Tracker::getGlobalConfiguration()
 {
   auto& tc = o2::its::TrackerParamConfig::Instance();
+  tc.printKeyValues(true, true);
   if (tc.useMatCorrTGeo) {
     mTraits->setCorrType(o2::base::PropagatorImpl<float>::MatCorrType::USEMatCorrTGeo);
   } else if (tc.useFastMaterial) {

Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx

@@ -22,6 +22,7 @@
 #include "DataFormatsTRD/TriggerRecord.h"
 #include "CommonDataFormat/IRFrame.h"
 #include "DetectorsBase/GRPGeomHelper.h"
+#include "ITStracking/TrackingConfigParam.h"
 
 namespace o2
 {
@@ -33,9 +34,11 @@ void ITSTrackingInterface::initialise()
   mRunVertexer = true;
   mCosmicsProcessing = false;
   std::vector<TrackingParameters> trackParams;
-
+  if (mMode == TrackingMode::Unset) {
+    mMode = (TrackingMode)(o2::its::TrackerParamConfig::Instance().trackingMode);
+    LOGP(info, "Tracking mode not set, trying to fetch it from configurable params to: {}", asString(mMode));
+  }
   if (mMode == TrackingMode::Async) {
-
     trackParams.resize(3);
     for (auto& param : trackParams) {
       param.ZBins = 64;
@@ -49,7 +52,6 @@ void ITSTrackingInterface::initialise()
     trackParams[2].CellDeltaTanLambdaSigma *= 4.;
     trackParams[2].MinTrackLength = 4;
     LOG(info) << "Initializing tracker in async. phase reconstruction with " << trackParams.size() << " passes";
-
   } else if (mMode == TrackingMode::Sync) {
     trackParams.resize(1);
     trackParams[0].ZBins = 64;
@@ -195,10 +197,7 @@ void ITSTrackingInterface::run(framework::ProcessingContext& pc)
         }
       }
       if (processingMask[iRof] && !selROF) { // passed selection in clusters and not in vertex multiplicity
-        LOG(debug) << fmt::format("ROF {} rejected by the vertex multiplicity selection [{},{}]",
-                                  iRof,
-                                  multEstConf.cutMultVtxLow,
-                                  multEstConf.cutMultVtxHigh);
+        LOGP(info, "ROF {} rejected by the vertex multiplicity selection [{},{}]", iRof, multEstConf.cutMultVtxLow, multEstConf.cutMultVtxHigh);
         processingMask[iRof] = selROF;
         cutVertexMult++;
       }
@@ -325,7 +324,7 @@ void ITSTrackingInterface::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
     return;
   }
   if (matcher == ConcreteDataMatcher("GLO", "MEANVERTEX", 0)) {
-    LOGP(info, "mean vertex acquired");
+    LOGP(info, "Mean vertex acquired");
     setMeanVertex((const o2::dataformats::MeanVertexObject*)obj);
     return;
   }

Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx

@@ -67,6 +67,7 @@ float Vertexer::clustersToVerticesHybrid(std::function<void(std::string s)> logg
 void Vertexer::getGlobalConfiguration()
 {
   auto& vc = o2::its::VertexerParamConfig::Instance();
+  vc.printKeyValues(true, true);
   auto& grc = o2::its::GpuRecoParamConfig::Instance();
 
   VertexingParameters verPar;
@@ -90,8 +91,7 @@ void Vertexer::getGlobalConfiguration()
   verPar.PhiBins = vc.PhiBins;
 
   TimeFrameGPUParameters tfGPUpar;
-  tfGPUpar.maxGPUMemoryGB = grc.maxGPUMemoryGB;
-  tfGPUpar.maxVerticesCapacity = grc.maxVerticesCapacity;
+  // tfGPUpar.nROFsPerChunk = grc.nROFsPerChunk;
 
   mTraits->updateVertexingParameters(verPar, tfGPUpar);
 }

Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h

@@ -23,13 +23,11 @@
 
 #include "ITStracking/TrackingInterface.h"
 
-#include "GPUO2Interface.h"
-#include "GPUReconstruction.h"
-#include "GPUChainITS.h"
-#include "CommonUtils/StringUtils.h"
-#include "TStopwatch.h"
+#include "GPUDataTypes.h"
 #include "DetectorsBase/GRPGeomHelper.h"
 
+#include "TStopwatch.h"
+
 namespace o2
 {
 namespace its
@@ -41,9 +39,9 @@ class TrackerDPL : public framework::Task
   TrackerDPL(std::shared_ptr<o2::base::GRPGeomRequest> gr,
              bool isMC,
              int trgType,
-             const TrackingMode& trMode,
+             const TrackingMode& trMode = TrackingMode::Unset,
              const bool overrBeamEst = false,
-             o2::gpu::GPUDataTypes::DeviceType dType = o2::gpu::GPUDataTypes::DeviceType::CPU);
+             gpu::GPUDataTypes::DeviceType dType = gpu::GPUDataTypes::DeviceType::CPU);
   ~TrackerDPL() override = default;
   void init(framework::InitContext& ic) final;
   void run(framework::ProcessingContext& pc) final;
@@ -60,7 +58,8 @@ class TrackerDPL : public framework::Task
   TStopwatch mTimer;
 };
 
-framework::DataProcessorSpec getTrackerSpec(bool useMC, bool useGeom, int useTrig, const std::string& trModeS, const bool overrBeamEst, o2::gpu::GPUDataTypes::DeviceType dType);
+using o2::its::TrackingMode;
+framework::DataProcessorSpec getTrackerSpec(bool useMC, bool useGeom, int useTrig, const std::string& trMode, const bool overrBeamEst = false, gpu::GPUDataTypes::DeviceType dType = gpu::GPUDataTypes::DeviceType::CPU);
 
 } // namespace its
 } // namespace o2