SIM: Adding experiment versions and dynamic library loading (#13148)

* SIM: Add detectorList option:

Gaining more flexibility with detector geometries.

(a) Allows to select a detector list
from a pre-defined list of ALICE2, ALICE2.1, ALICE3 (corresponding to Run3, Run4, Run5).
Arguments for -m, --skipModules, --readoutDetectors will then be checked against that list.

`o2-sim -n 1 --detectorList ALICE2.1 -m IT3  -o bar2`

(b) Allows to define a custom detector list "MyList" in a JSON file, and following the notation:

`o2-sim --detectorList MyList:${PWD}/myDetector.json`

* SIM: Add dynamic detector library loading

This commit also adds the possibility to instantiate/construct detector modules as a dynamic plugin.
The benefit is that we no longer need to link all possible detector libraries into the simulation executable.

(cherry picked from commit be5cdb9)

Author: f3sch

Common/SimConfig/CMakeLists.txt

@@ -12,8 +12,11 @@
 o2_add_library(SimConfig
                SOURCES src/SimConfig.cxx
                        src/SimParams.cxx
+                       src/SimDLLoader.cxx
                        src/SimUserDecay.cxx
-                       src/DigiParams.cxx src/G4Params.cxx
+                       src/DigiParams.cxx
+                       src/G4Params.cxx
+                       src/DetectorLists.cxx
                        src/MatMapParams.cxx
                        src/InteractionDiamondParam.cxx
                        src/GlobalProcessCutSimParam.cxx
@@ -25,10 +28,12 @@ o2_add_library(SimConfig
 o2_target_root_dictionary(SimConfig
                           HEADERS include/SimConfig/SimConfig.h
                                   include/SimConfig/SimParams.h
+                                  include/SimConfig/SimDLLoader.h
                                   include/SimConfig/SimUserDecay.h
                                   include/SimConfig/InteractionDiamondParam.h
-		                  include/SimConfig/DigiParams.h
+                                  include/SimConfig/DigiParams.h
                                   include/SimConfig/G4Params.h
+                                  include/SimConfig/DetectorLists.h
                                   include/SimConfig/GlobalProcessCutSimParam.h
                                   include/SimConfig/MatMapParams.h)
 

Common/SimConfig/include/SimConfig/DetectorLists.h

@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_DETECTORLISTS_H_
+#define O2_DETECTORLISTS_H_
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "Framework/Logger.h"
+
+namespace o2::conf
+{
+// Container defining different general evolutions of the ALICE experiment. Each
+// evolution is given a name and a list defining the names of the detectors and
+// passive elements present.
+using DetectorList_t = std::vector<std::string>;
+using DetectorMap_t = std::unordered_map<std::string, DetectorList_t>;
+
+// Parse the detector map from a JSON file.
+// Return false if parsing failed.
+bool parseDetectorMapfromJSON(const std::string& path, DetectorMap_t& map);
+
+// Print the DetetectorMap
+void printDetMap(const DetectorMap_t& map, const std::string& list = "");
+} // namespace o2::conf
+
+#endif // O2_DETECTORLISTS_H_

Common/SimConfig/include/SimConfig/SimConfig.h

@@ -48,43 +48,43 @@ enum class TimeStampMode {
 
 // configuration struct (which can be passed around)
 struct SimConfigData {
-  std::vector<std::string> mActiveModules;    // list of active modules
-  std::vector<std::string> mReadoutDetectors; // list of readout detectors
-  std::string mMCEngine;                      // chosen VMC engine
-  std::string mGenerator;                     // chosen VMC generator
-  std::string mTrigger;                       // chosen VMC generator trigger
-  unsigned int mNEvents;                      // number of events to be simulated
-  std::string mExtKinFileName;                // file name of external kinematics file (needed for ext kinematics generator)
-  std::string mEmbedIntoFileName;             // filename containing the reference events to be used for the embedding
-  unsigned int mStartEvent;                   // index of first event to be taken
-  float mBMax;                                // maximum for impact parameter sampling
-  bool mIsMT;                                 // chosen MT mode (Geant4 only)
-  std::string mOutputPrefix;                  // prefix to be used for output files
-  std::string mLogSeverity;                   // severity for FairLogger
-  std::string mLogVerbosity;                  // loglevel for FairLogger
-  std::string mKeyValueTokens;                // a string holding arbitrary sequence of key-value tokens
-                                              // Foo.parameter1=x,Bar.parameter2=y,Baz.paramter3=hello
-                                              // (can be used to **loosely** change any configuration parameter from command-line)
-  std::string mConfigFile;                    // path to a JSON or INI config file (file extension is required to determine type).
-                                              // values within the config file will override values set in code by the param classes
-                                              // but will themselves be overridden by any values given in mKeyValueTokens.
-  int mPrimaryChunkSize;                      // defining max granularity for input primaries of a sim job
-  int mInternalChunkSize;                     //
-  ULong_t mStartSeed;                         // base for random number seeds
-  int mSimWorkers = 1;                        // number of parallel sim workers (when it applies)
-  bool mFilterNoHitEvents = false;            // whether to filter out events not leaving any response
-  std::string mCCDBUrl;                       // the URL where to find CCDB
-  uint64_t mTimestamp;                        // timestamp in ms to anchor transport simulation to
+  std::vector<std::string> mActiveModules;            // list of active modules
+  std::vector<std::string> mReadoutDetectors;         // list of readout detectors
+  std::string mMCEngine;                              // chosen VMC engine
+  std::string mGenerator;                             // chosen VMC generator
+  std::string mTrigger;                               // chosen VMC generator trigger
+  unsigned int mNEvents;                              // number of events to be simulated
+  std::string mExtKinFileName;                        // file name of external kinematics file (needed for ext kinematics generator)
+  std::string mEmbedIntoFileName;                     // filename containing the reference events to be used for the embedding
+  unsigned int mStartEvent;                           // index of first event to be taken
+  float mBMax;                                        // maximum for impact parameter sampling
+  bool mIsMT;                                         // chosen MT mode (Geant4 only)
+  std::string mOutputPrefix;                          // prefix to be used for output files
+  std::string mLogSeverity;                           // severity for FairLogger
+  std::string mLogVerbosity;                          // loglevel for FairLogger
+  std::string mKeyValueTokens;                        // a string holding arbitrary sequence of key-value tokens
+                                                      // Foo.parameter1=x,Bar.parameter2=y,Baz.paramter3=hello
+                                                      // (can be used to **loosely** change any configuration parameter from command-line)
+  std::string mConfigFile;                            // path to a JSON or INI config file (file extension is required to determine type).
+                                                      // values within the config file will override values set in code by the param classes
+                                                      // but will themselves be overridden by any values given in mKeyValueTokens.
+  unsigned int mPrimaryChunkSize;                     // defining max granularity for input primaries of a sim job
+  int mInternalChunkSize;                             //
+  ULong_t mStartSeed;                                 // base for random number seeds
+  int mSimWorkers = 1;                                // number of parallel sim workers (when it applies)
+  bool mFilterNoHitEvents = false;                    // whether to filter out events not leaving any response
+  std::string mCCDBUrl;                               // the URL where to find CCDB
+  uint64_t mTimestamp;                                // timestamp in ms to anchor transport simulation to
   TimeStampMode mTimestampMode = TimeStampMode::kNow; // telling of timestamp was given as option or defaulted to now
-  int mRunNumber = -1;                        // ALICE run number (if set != -1); the timestamp should be compatible
-  int mField;                                 // L3 field setting in kGauss: +-2,+-5 and 0
-  SimFieldMode mFieldMode = SimFieldMode::kDefault; // uniform magnetic field
-  bool mAsService = false;                    // if simulation should be run as service/deamon (does not exit after run)
-  bool mNoGeant = false;                      // if Geant transport should be turned off (when one is only interested in the generated events)
-  bool mIsUpgrade = false;                    // true if the simulation is for Run 5
-  std::string mFromCollisionContext = "";     // string denoting a collision context file; If given, this file will be used to determine number of events
-  bool mForwardKine = false;                  // true if tracks and event headers are to be published on a FairMQ channel (for reading by other consumers)
-  bool mWriteToDisc = true;                   // whether we write simulation products (kine, hits) to disc
+  int mRunNumber = -1;                                // ALICE run number (if set != -1); the timestamp should be compatible
+  int mField;                                         // L3 field setting in kGauss: +-2,+-5 and 0
+  SimFieldMode mFieldMode = SimFieldMode::kDefault;   // uniform magnetic field
+  bool mAsService = false;                            // if simulation should be run as service/deamon (does not exit after run)
+  bool mNoGeant = false;                              // if Geant transport should be turned off (when one is only interested in the generated events)
+  bool mIsUpgrade = false;                            // true if the simulation is for Run 5
+  std::string mFromCollisionContext = "";             // string denoting a collision context file; If given, this file will be used to determine number of events
+  bool mForwardKine = false;                          // true if tracks and event headers are to be published on a FairMQ channel (for reading by other consumers)
+  bool mWriteToDisc = true;                           // whether we write simulation products (kine, hits) to disc
   VertexMode mVertexMode = VertexMode::kDiamondParam; // by default we should use die InteractionDiamond parameter
 
   ClassDefNV(SimConfigData, 4);
@@ -140,6 +140,7 @@ class SimConfig
   // static helper functions to determine list of active / readout modules
   // can also be used from outside
   static void determineActiveModules(std::vector<std::string> const& input, std::vector<std::string> const& skipped, std::vector<std::string>& active, bool isUpgrade = false);
+  static bool determineActiveModulesList(const std::string& version, std::vector<std::string> const& input, std::vector<std::string> const& skipped, std::vector<std::string>& active);
   static void determineReadoutDetectors(std::vector<std::string> const& active, std::vector<std::string> const& enabledRO, std::vector<std::string> const& skippedRO, std::vector<std::string>& finalRO);
 
   // helper to parse field option
@@ -179,6 +180,9 @@ class SimConfig
  private:
   SimConfigData mConfigData; //!
 
+  // Filter out skipped elements in the list
+  static bool filterSkippedElements(std::vector<std::string>& elements, std::vector<std::string> const& skipped);
+
   // adjust/overwrite some option settings when collision context is used
   void adjustFromCollContext(std::string const& collcontextfile, std::string const& prefix);
 
@@ -206,9 +210,9 @@ struct SimReconfigData {
   std::string configFile; // path to a JSON or INI config file (file extension is required to determine type).
   // values within the config file will override values set in code by the param classes
   // but will themselves be overridden by any values given in mKeyValueTokens.
-  unsigned int primaryChunkSize; // defining max granularity for input primaries of a sim job
-  ULong_t startSeed;             // base for random number seeds
-  bool stop;                     // to shut down the service
+  unsigned int primaryChunkSize;          // defining max granularity for input primaries of a sim job
+  ULong_t startSeed;                      // base for random number seeds
+  bool stop;                              // to shut down the service
   std::string mFromCollisionContext = ""; // string denoting a collision context file; If given, this file will be used to determine number of events
 
   ClassDefNV(SimReconfigData, 1);

Common/SimConfig/include/SimConfig/SimDLLoader.h

@@ -0,0 +1,27 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef SIMDLLOADER_H_
+#define SIMDLLOADER_H_
+
+#include "CommonUtils/DLLoaderBase.h"
+
+namespace o2::conf
+{
+
+class SimDLLoader : public o2::utils::DLLoaderBase<SimDLLoader>
+{
+  O2DLLoaderDef(SimDLLoader)
+};
+
+} // namespace o2::conf
+
+#endif // SIMDLLOADER_H_

Common/SimConfig/src/DetectorLists.cxx

@@ -0,0 +1,78 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "SimConfig/DetectorLists.h"
+#include <fairlogger/Logger.h>
+#include <rapidjson/document.h>
+#include <rapidjson/error/en.h>
+#include <rapidjson/istreamwrapper.h>
+#include <fstream>
+
+namespace o2::conf
+{
+
+bool parseDetectorMapfromJSON(const std::string& path, DetectorMap_t& map)
+{
+  // Parse JSON file to build map
+  std::ifstream fileStream(path, std::ios::in);
+  if (!fileStream.is_open()) {
+    LOGP(error, "Cannot open '{}'!", path);
+    return false;
+  }
+  rapidjson::IStreamWrapper isw(fileStream);
+  rapidjson::Document doc;
+  doc.ParseStream(isw);
+  if (doc.HasParseError()) {
+    LOGP(error, "Error parsing provided json file '{}':", path);
+    LOGP(error, "  - Error -> {}", rapidjson::GetParseError_En(doc.GetParseError()));
+    LOGP(error, "  - Offset -> {}", doc.GetErrorOffset());
+    return false;
+  }
+
+  // Clear and rebuild map
+  map.clear();
+  try {
+    for (auto verItr = doc.MemberBegin(); verItr != doc.MemberEnd(); ++verItr) {
+      const auto& version = verItr->name.GetString();
+      DetectorList_t list;
+      const auto& elements = doc[version];
+      for (const auto& ele : elements.GetArray()) {
+        list.emplace_back(ele.GetString());
+      }
+      map.emplace(version, list);
+    }
+  } catch (const std::exception& e) {
+    LOGP(error, "Failed to build detector map from file '{}' with '{}'", path, e.what());
+    return false;
+  }
+
+  return true;
+}
+
+void printDetMap(const DetectorMap_t& map, const std::string& list)
+{
+  if (list.empty()) {
+    LOGP(error, "List of all available versions including their detectors:");
+    for (int i{0}; const auto& [version, elements] : map) {
+      LOGP(error, " - {: >2d}. {}:", i++, version);
+      for (int j{0}; const auto& element : elements) {
+        LOGP(error, "\t\t* {: >2d}.\t{}", j++, element);
+      }
+    }
+  } else {
+    LOGP(error, "List of available modules for version {}:", list);
+    for (int j{0}; const auto& element : map.at(list)) {
+      LOGP(error, "\t* {: >2d}.\t{}", j++, element);
+    }
+  }
+}
+
+} // namespace o2::conf