Merge branch 'AliceO2Group:dev' into new-detector4

Author: pkurash

CCDB/src/CcdbApi.cxx

@@ -1461,7 +1461,7 @@ std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& p
 
   if (!mSnapshotCachePath.empty()) {
     // protect this sensitive section by a multi-process named semaphore
-    auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, path);
+    auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath + std::string("_headers"), path);
 
     std::string logfile = mSnapshotCachePath + "/log";
     std::fstream out(logfile, ios_base::out | ios_base::app);

Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/ThresholdCalibratorSpec.h

@@ -148,7 +148,6 @@ class ITSThresholdCalibrator : public Task
 
   // Hash tables to store the hit and threshold information per pixel
   std::map<short int, std::map<int, std::vector<std::vector<std::vector<unsigned short int>>>>> mPixelHits;
-  std::map<short int, std::deque<short int>> mForbiddenRows;
   // Unordered map for saving sum of values (thr/ithr/vcasn) for avg calculation
   std::map<short int, std::array<long int, 6>> mThresholds;
   // Map including PixID for noisy pixels

Detectors/ITSMFT/ITS/workflow/src/ThresholdCalibratorSpec.cxx

@@ -83,6 +83,8 @@ void NoiseCalibratorSpec::run(ProcessingContext& pc)
       } else {
         LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
         sendOutputCcdbDcs(pc.outputs());
+        LOG(info) << "Sending an object to Production-CCDBMerge";
+        sendOutputCcdbMerge(pc.outputs());
       }
       pc.services().get<ControlService>().readyToQuit(mStopMeOnly ? QuitRequest::Me : QuitRequest::All);
     }
@@ -105,6 +107,8 @@ void NoiseCalibratorSpec::run(ProcessingContext& pc)
       } else {
         LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
         sendOutputCcdbDcs(pc.outputs());
+        LOG(info) << "Sending an object to Production-CCDBMerge";
+        sendOutputCcdbMerge(pc.outputs());
       }
       pc.services().get<ControlService>().readyToQuit(mStopMeOnly ? QuitRequest::Me : QuitRequest::All);
     }

Detectors/ITSMFT/MFT/calibration/src/NoiseCalibratorSpec.cxx

@@ -66,6 +66,7 @@ enum class CDBType {
   CalSAC1,              ///< I_1(t) = <I(r,\phi,t) / I_0(r,\phi)>_{r,\phi}
   CalSACDelta,          ///< \Delta I(r,\phi,t) = I(r,\phi,t) / ( I_0(r,\phi) * I_1(t) )
   CalSACFourier,        ///< Fourier coefficients of CalSAC1
+  CalSAC,               ///< CalSAC0, CalSAC1 and CalSACDelta
                         ///
   CalITPC0,             ///< 2D average TPC clusters for longer time interval
   CalITPC1,             ///< 1D integrated TPC clusters
@@ -133,6 +134,7 @@ const std::unordered_map<CDBType, const std::string> CDBTypeMap{
   {CDBType::CalSAC0, "TPC/Calib/SAC_0"},
   {CDBType::CalSAC1, "TPC/Calib/SAC_1"},
   {CDBType::CalSACDelta, "TPC/Calib/SAC_DELTA"},
+  {CDBType::CalSAC, "TPC/Calib/SAC"},
   {CDBType::CalSACFourier, "TPC/Calib/SAC_FOURIER"},
   // ITPCCs
   {CDBType::CalITPC0, "TPC/Calib/ITPCC_0"},

Detectors/TPC/base/include/TPCBase/CDBTypes.h

@@ -93,10 +93,8 @@ constexpr float fixedSizeToFloat(uint32_t value)
 /// write values of map to fileName
 ///
 template <typename DataMap>
-void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false)
+void writeValues(std::ostream& str, const DataMap& map, bool onlyFilled = false)
 {
-  std::ofstream str(fileName.data(), std::ofstream::out);
-
   for (const auto& [linkInfo, data] : map) {
     if (onlyFilled) {
       if (!std::accumulate(data.begin(), data.end(), uint32_t(0))) {
@@ -117,6 +115,13 @@ void writeValues(const std::string_view fileName, const DataMap& map, bool onlyF
   }
 }
 
+template <typename DataMap>
+void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false)
+{
+  std::ofstream str(fileName.data(), std::ofstream::out);
+  writeValues(str, map, onlyFilled);
+}
+
 template <class T>
 struct is_map {
   static constexpr bool value = false;
@@ -126,7 +131,8 @@ template <class Key, class Value>
 struct is_map<std::map<Key, Value>> {
   static constexpr bool value = true;
 };
-/// fill cal pad object from HV data map
+
+/// fill cal pad object from HW data map
 /// TODO: Function to be tested
 template <typename DataMap, uint32_t SignificantBitsT = 0>
 typename std::enable_if_t<is_map<DataMap>::value, void>
@@ -251,6 +257,7 @@ o2::tpc::CalDet<float> getCalPad(const std::string_view fileName, const std::str
 /// \param minADCROCType can be either one value for all ROC types, or {IROC, OROC}, or {IROC, OROC1, OROC2, OROC3}
 std::unordered_map<std::string, CalPad> preparePedestalFiles(const CalPad& pedestals, const CalPad& noise, std::vector<float> sigmaNoiseROCType = {3, 3, 3, 3}, std::vector<float> minADCROCType = {2, 2, 2, 2}, float pedestalOffset = 0, bool onlyFilled = false, bool maskBad = true, float noisyChannelThreshold = 1.5, float sigmaNoiseNoisyChannels = 4, float badChannelThreshold = 6, bool fixedSize = false);
 
+DataMapU32 getDataMap(const CalPad& calPad);
 } // namespace o2::tpc::cru_calib_helpers
 
 #endif

Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h

@@ -130,9 +130,6 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(
   pedestalsThreshold["PedestalsPhys"] = CalPad("Pedestals");
   pedestalsThreshold["ThresholdMapPhys"] = CalPad("ThresholdMap");
 
-  auto& pedestalsCRU = pedestalsThreshold["Pedestals"];
-  auto& thresholdCRU = pedestalsThreshold["ThresholdMap"];
-
   // ===| prepare values |===
   for (size_t iroc = 0; iroc < pedestals.getData().size(); ++iroc) {
     const ROC roc(iroc);
@@ -179,7 +176,7 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(
       }
 
       float noise = std::abs(rocNoise.getValue(ipad)); // it seems with the new fitting procedure, the noise can also be negative, since in gaus sigma is quadratic
-      float noiseCorr = noise - (0.847601 + 0.031514 * traceLength);
+      const float noiseCorr = noise - (0.847601 + 0.031514 * traceLength);
       if ((pedestal <= 0) || (pedestal > 150) || (noise <= 0) || (noise > 50)) {
         LOGP(info, "Bad pedestal or noise value in ROC {:2}, CRU {:3}, fec in CRU: {:2}, SAMPA: {}, channel: {:2}, pedestal: {:.4f}, noise {:.4f}", iroc, cruID, fecInPartition, sampa, sampaChannel, pedestal, noise);
         if (maskBad) {
@@ -230,3 +227,48 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(
 
   return pedestalsThreshold;
 }
+
+cru_calib_helpers::DataMapU32 cru_calib_helpers::getDataMap(const CalPad& calPad)
+{
+  const auto& mapper = Mapper::instance();
+
+  DataMapU32 dataMap;
+
+  for (size_t iroc = 0; iroc < calPad.getData().size(); ++iroc) {
+    const ROC roc(iroc);
+
+    const auto& calRoc = calPad.getCalArray(iroc);
+
+    const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0;
+
+    // skip empty ROCs
+    if (!(std::abs(calRoc.getSum()) > 0)) {
+      continue;
+    }
+
+    // loop over pads
+    for (size_t ipad = 0; ipad < calRoc.getData().size(); ++ipad) {
+      const int globalPad = ipad + padOffset;
+      const FECInfo& fecInfo = mapper.fecInfo(globalPad);
+      const CRU cru = mapper.getCRU(roc.getSector(), globalPad);
+      const uint32_t region = cru.region();
+      const int cruID = cru.number();
+      const int sampa = fecInfo.getSampaChip();
+      const int sampaChannel = fecInfo.getSampaChannel();
+
+      const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()];
+      const int nFECs = partInfo.getNumberOfFECs();
+      const int fecOffset = (nFECs + 1) / 2;
+      const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset();
+      const int dataWrapperID = fecInPartition >= fecOffset;
+      const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12;
+
+      const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2);
+
+      const auto value = calRoc.getValue(ipad);
+      dataMap[LinkInfo(cruID, globalLinkID)][hwChannel] = floatToFixedSize(value);
+    }
+  }
+
+  return dataMap;
+}

Detectors/TPC/base/src/CRUCalibHelpers.cxx

@@ -402,6 +402,14 @@ struct SACDelta {
   std::array<IDCDelta<DataT>, SIDES> mSACDelta{};
 };
 
+/// container to be written to CCDB
+template <typename DataT>
+struct SAC {
+  SACZero mSACZero{};
+  SACOne mSACOne{};
+  SACDelta<DataT> mSACDelta{};
+};
+
 struct FourierCoeffSAC {
   std::array<FourierCoeff, SIDES> mCoeff{};
 };

Detectors/TPC/calibration/include/TPCCalibration/IDCContainer.h

@@ -60,72 +60,17 @@ void preparePedestalFiles(const std::string_view pedestalFile, std::string outpu
     f.GetObject("Noise", calNoise);
   }
 
-  DataMapU32 pedestalValues;
-  DataMapU32 thresholdlValues;
-  DataMapU32 pedestalValuesPhysics;
-  DataMapU32 thresholdlValuesPhysics;
-
   auto pedestalsThreshold = preparePedestalFiles(*calPedestal, *calNoise, sigmaNoiseROCType, minADCROCType, pedestalOffset, onlyFilled, maskBad, noisyChannelThreshold, sigmaNoiseNoisyChannels, badChannelThreshold);
 
-  // ===| prepare values |===
-  for (size_t iroc = 0; iroc < calPedestal->getData().size(); ++iroc) {
-    const ROC roc(iroc);
-
-    const auto& rocPedestal = calPedestal->getCalArray(iroc);
-    const auto& rocNoise = calNoise->getCalArray(iroc);
-    auto& rocOut = output.getCalArray(iroc);
+  const auto& pedestals = pedestalsThreshold["Pedestals"];
+  const auto& thresholds = pedestalsThreshold["ThresholdMap"];
+  const auto& pedestalsPhys = pedestalsThreshold["PedestalsPhys"];
+  const auto& thresholdsPhys = pedestalsThreshold["ThresholdMapPhys"];
 
-    const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0;
-
-    // skip empty
-    if (!(std::abs(rocPedestal.getSum() + rocNoise.getSum()) > 0)) {
-      continue;
-    }
-
-    // loop over pads
-    for (size_t ipad = 0; ipad < rocPedestal.getData().size(); ++ipad) {
-      const int globalPad = ipad + padOffset;
-      const FECInfo& fecInfo = mapper.fecInfo(globalPad);
-      const CRU cru = mapper.getCRU(roc.getSector(), globalPad);
-      const uint32_t region = cru.region();
-      const int cruID = cru.number();
-      const int sampa = fecInfo.getSampaChip();
-      const int sampaChannel = fecInfo.getSampaChannel();
-      // int globalLinkID = fecInfo.getIndex();
-
-      const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()];
-      const int nFECs = partInfo.getNumberOfFECs();
-      const int fecOffset = (nFECs + 1) / 2;
-      const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset();
-      const int dataWrapperID = fecInPartition >= fecOffset;
-      const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12;
-
-      const auto pedestal = pedestalsThreshold["Pedestals"].getCalArray(iroc).getValue(ipad);
-      const auto threshold = pedestalsThreshold["ThresholdMap"].getCalArray(iroc).getValue(ipad);
-      const auto pedestalHighNoise = pedestalsThreshold["PedestalsPhys"].getCalArray(iroc).getValue(ipad);
-      const auto thresholdHighNoise = pedestalsThreshold["ThresholdMapPhys"].getCalArray(iroc).getValue(ipad);
-
-      const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2);
-      // for debugging
-      // printf("%4d %4d %4d %4d %4d: %u\n", cru.number(), globalLinkID, hwChannel, fecInfo.getSampaChip(), fecInfo.getSampaChannel(), getADCValue(pedestal));
-
-      // default thresholds
-      const auto adcPedestal = floatToFixedSize(pedestal);
-      const auto adcThreshold = floatToFixedSize(threshold);
-      pedestalValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestal;
-      thresholdlValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThreshold;
-
-      // higher thresholds for physics data taking
-      const auto adcPedestalPhysics = floatToFixedSize(pedestalHighNoise);
-      const auto adcThresholdPhysics = floatToFixedSize(thresholdHighNoise);
-      pedestalValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestalPhysics;
-      thresholdlValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThresholdPhysics;
-      // for debugging
-      // if(!(std::abs(pedestal - fixedSizeToFloat(adcPedestal)) <= 0.5 * 0.25)) {
-      // printf("%4d %4d %4d %4d %4d: %u %.2f %.4f %.4f\n", cru.number(), globalLinkID, hwChannel, sampa, sampaChannel, adcPedestal, fixedSizeToFloat(adcPedestal), pedestal, pedestal - fixedSizeToFloat(adcPedestal));
-      //}
-    }
-  }
+  auto pedestalValues = getDataMap(pedestals);
+  auto thresholdlValues = getDataMap(thresholds);
+  auto pedestalValuesPhysics = getDataMap(pedestalsPhys);
+  auto thresholdlValuesPhysics = getDataMap(thresholdsPhys);
 
   // text files
   const auto outFilePedestalTXT(outputDir + "/pedestal_values.txt");

Detectors/TPC/calibration/macro/preparePedestalFiles.C

@@ -549,7 +549,8 @@ void CalibdEdx::finalize(const bool useGausFits)
     fitter.SetFormula("1");
     mCalib.setDims(0);
   }
-  LOGP(info, "Fitting {}D dE/dx correction for GEM stacks with gaussian fits {}", mCalib.getDims(), useGausFits);
+  LOGP(info, "Fitting {}D dE/dx correction for GEM stacks with gaussian fits {}, minStackEntries {}, m2DThreshold {}, m1DThreshold {}, mFitSnp {}",
+       mCalib.getDims(), useGausFits, entries, m2DThreshold, m1DThreshold, mFitSnp);
 
   // if entries below minimum sector threshold, integrate all sectors
   if (mCalib.getDims() == 0 || entries >= mSectorThreshold) {