Add possibility to specify a manual reject list for MID

Author: dstocco

Detectors/MUON/MID/Calibration/macros/README.md

@@ -60,6 +60,53 @@ root -l
 .x build_rejectlist.C+(1716436103391,1721272208000,"localhost:8083")
 ```
 
+### Add custom bad channels
+
+The macro `build_rejectlist.C` scans the QCDB and the CCDB in search of issues.
+However, the QCDB flag is based on local boards with empty signals.
+It can happen that a local board is problematic, but not completely dead and, therefore, it is not correctly spotted by the macro.
+It is therefore important to have a way to add the issues by hand.
+This can be done with a json file in the form:
+
+```json
+{
+    "startRun": 557251,
+    "endRun": 557926,
+    "rejectList": [
+        {
+            "deId": 4,
+            "columnId": 2,
+            "patterns": [
+                "0x0",
+                "0xFFFF",
+                "0x0",
+                "0x0",
+                "0x0"
+            ]
+        },
+        {
+            "deId": 13,
+            "columnId": 2,
+            "patterns": [
+                "0x0",
+                "0xFFFF",
+                "0x0",
+                "0x0",
+                "0x0"
+            ]
+        }
+    ]
+}
+```
+
+The path to the file is then given to the macro with:
+
+```shell
+.x build_rejectlist.C+(1726299038000,1727386238000,"http://localhost:8083","http://alice-ccdb.cern.ch","http://localhost:8080","rejectlist.json")
+```
+
+The macro will then merge the manual reject list from the file with the reject list that it finds by scanning the QCDB and CCDB.
+
 ## Running the local CCDB
 
 The local CCDB server can be easily built through alibuild.

Detectors/MUON/MID/Calibration/macros/build_rejectlist.C

@@ -21,6 +21,8 @@
 #include <vector>
 #include <limits>
 #include <algorithm>
+#include "rapidjson/document.h"
+#include "rapidjson/istreamwrapper.h"
 #include "TCanvas.h"
 #include "TH1.h"
 #include "TGraph.h"
@@ -29,15 +31,25 @@
 #include "DataFormatsParameters/GRPECSObject.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DataFormatsMID/ColumnData.h"
+#include "MIDBase/ColumnDataHandler.h"
 #include "MIDGlobalMapping/ExtendedMappingInfo.h"
 #include "MIDGlobalMapping/GlobalMapper.h"
 #include "MIDFiltering/ChannelMasksHandler.h"
+
+// ...
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 #include "CCDB/BasicCCDBManager.h"
 #endif
 
 static const std::string sPathQCQuality = "qc/MID/MO/MIDQuality/Trends/global/MIDQuality/MIDQuality";
 
+/// @brief  Reject list object
+struct RejectListStruct {
+  long start = 0;                                /// Start validity
+  long end = 0;                                  /// End validity
+  std::vector<o2::mid::ColumnData> rejectList{}; /// Bad channels
+};
+
 /// @brief Get timestamp in milliseconds
 /// @param timestamp Input timestamp (in s or ms)
 /// @return Timestamp in ms
@@ -168,25 +180,38 @@ std::vector<o2::mid::ColumnData> getRejectList(std::vector<o2::mid::ColumnData>
   return badChannels;
 }
 
+/// @brief Gets the run duration with a safety marging
+/// @param ccdbApi CCDB api
+/// @param marging margin in milliseconds
+/// @return Pair with the timestamps of start-margin and end+margin for the run
+std::pair<int64_t, int64_t> getRunDuration(const o2::ccdb::CcdbApi& ccdbApi, int runNumber, int64_t margin = 120000)
+{
+  auto runRange = o2::ccdb::BasicCCDBManager::getRunDuration(ccdbApi, runNumber);
+  runRange.first -= margin;  // Subtract margin
+  runRange.second += margin; // Add margin
+  return runRange;
+}
+
 /// @brief Builds the reject list for the selected timestamp
 /// @param timestamp Timestamp for query
 /// @param qcdbApi QCDB api
 /// @param ccdbApi CCDB api
 /// @param outCCDBApi api of the CCDB where the reject list will be uploaded
 /// @return Reject list
-std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const o2::ccdb::CcdbApi& qcdbApi, const o2::ccdb::CcdbApi& ccdbApi, const o2::ccdb::CcdbApi& outCCDBApi)
+RejectListStruct build_rejectlist(long timestamp, const o2::ccdb::CcdbApi& qcdbApi, const o2::ccdb::CcdbApi& ccdbApi)
 {
   std::map<std::string, std::string> metadata;
+  RejectListStruct rl;
   auto* qcQuality = qcdbApi.retrieveFromTFileAny<TCanvas>(sPathQCQuality, metadata, getTSMS(timestamp));
   if (!qcQuality) {
     std::cerr << "Cannot find QC quality for " << tsToString(timestamp) << std::endl;
-    return {};
+    return rl;
   }
   // Find the first and last timestamp where the quality was bad (if any)
   auto badTSRange = findTSRange(qcQuality);
   if (badTSRange.second == 0) {
     std::cout << "All good" << std::endl;
-    return {};
+    return rl;
   }
   // Search for the last timestamp for which the run quality was good
   auto goodTSRange = findTSRange(qcQuality, false);
@@ -196,18 +221,15 @@ std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const o2::ccdb
   if (!grpecs.isDetReadOut(o2::detectors::DetID::MID)) {
     std::cout << "Error: we are probably reading a parallel run" << std::endl;
     grpecs.print();
-    return {};
+    return rl;
   }
   if (grpecs.getRunType() != o2::parameters::GRPECS::PHYSICS) {
     std::cout << "This is not a physics run: skip" << std::endl;
     grpecs.print();
-    return {};
+    return rl;
   }
 
-  auto runRange = o2::ccdb::BasicCCDBManager::getRunDuration(ccdbApi, grpecs.getRun());
-  long margin = 120000;      // Add a two minutes safety margin
-  runRange.first -= margin;  // Subtract margin
-  runRange.second += margin; // Add margin
+  auto runRange = getRunDuration(ccdbApi, grpecs.getRun());
 
   // Search for hits histogram in the period where the QC quality was bad
   auto tsVector = findObjectsTSInPeriod(badTSRange.first, badTSRange.second, qcdbApi, "qc/MID/MO/QcTaskMIDDigits/Hits");
@@ -221,15 +243,15 @@ std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const o2::ccdb
   auto infos = gm.buildStripsInfo();
   auto badChannels = findBadChannels(occupancy, infos);
   auto badChannelsCCDB = *ccdbApi.retrieveFromTFileAny<std::vector<o2::mid::ColumnData>>("MID/Calib/BadChannels", metadata, getTSMS(timestamp));
-  auto rejectList = getRejectList(badChannels, badChannelsCCDB);
-  if (rejectList.empty()) {
+  rl.rejectList = getRejectList(badChannels, badChannelsCCDB);
+  if (rl.rejectList.empty()) {
     std::cout << "Warning: reject list was empty. It probably means that an entire board is already masked in calibration for run " << grpecs.getRun() << std::endl;
-    return {};
+    return rl;
   }
 
   // Print some useful information
   std::cout << "Reject list:" << std::endl;
-  for (auto& col : rejectList) {
+  for (auto& col : rl.rejectList) {
     std::cout << col << std::endl;
   }
   std::cout << "Run number: " << grpecs.getRun() << std::endl;
@@ -239,41 +261,120 @@ std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const o2::ccdb
   std::cout << "Bad:       " << timeRangeToString(badTSRange.first, badTSRange.second) << std::endl;
 
   // Set the start of the reject list to the last timestamp in which the occupancy was ok
-  auto startRL = goodTSRange.second;
+  rl.start = goodTSRange.second;
   if (goodTSRange.first == 0) {
     // If the quality was bad for the full run, set the start of the reject list to the SOR
     std::cout << "CAVEAT: no good TS found. Will use SOT instead" << std::endl;
-    startRL = runRange.first;
+    rl.start = runRange.first;
   }
   // Set the end of the reject list to the end of run
-  auto endRL = runRange.second;
-  // Ask if you want to upload the object to the CCDB
-  std::cout << "Upload reject list with validity: " << startRL << " - " << endRL << " to " << outCCDBApi.getURL() << "? [y/n]" << std::endl;
-  std::string answer;
-  std::cin >> answer;
-  if (answer == "y") {
-    std::cout << "Storing RejectList valid from " << startRL << " to " << endRL << std::endl;
-    outCCDBApi.storeAsTFileAny(&rejectList, "MID/Calib/RejectList", metadata, startRL, endRL);
+  rl.end = runRange.second;
+  return rl;
+}
+
+/// @brief Loads the reject list from a json file
+/// @param ccdbApi CCDB api
+/// @param filename json filename
+/// @return Reject list structure
+RejectListStruct load_from_json(const o2::ccdb::CcdbApi& ccdbApi, const char* filename = "rejectlist.json")
+{
+  // Open the JSON file
+  std::cout << "Reading reject list from file " << filename << std::endl;
+  RejectListStruct rl;
+  std::ifstream inFile(filename);
+  if (!inFile.is_open()) {
+    std::cerr << "Could not open the file!" << std::endl;
+    return rl;
   }
-  return rejectList;
+
+  // Create an IStreamWrapper for file input stream
+  rapidjson::IStreamWrapper isw(inFile);
+
+  rapidjson::Document doc;
+  if (doc.ParseStream(isw).HasParseError()) {
+    std::cerr << "Problem parsing " << filename << std::endl;
+    return rl;
+  }
+  auto startRange = getRunDuration(ccdbApi, doc["startRun"].GetInt());
+  auto endRange = getRunDuration(ccdbApi, doc["endRun"].GetInt());
+  rl.start = startRange.first;
+  rl.end = endRange.second;
+  std::cout << "Manual RL validity: " << timeRangeToString(rl.start, rl.end) << std::endl;
+  auto rlArray = doc["rejectList"].GetArray();
+  for (auto& ar : rlArray) {
+    o2::mid::ColumnData col;
+    col.deId = ar["deId"].GetInt();
+    col.columnId = ar["columnId"].GetInt();
+    auto patterns = ar["patterns"].GetArray();
+    for (size_t iar = 0; iar < 5; ++iar) {
+      col.patterns[iar] = std::strtol(patterns[iar].GetString(), NULL, 16);
+    }
+    rl.rejectList.emplace_back(col);
+    std::cout << col << std::endl;
+  }
+  return rl;
 }
 
-/// @brief Builds the reject list for the selected timestamp
-/// @param timestamp Timestamp for query
-/// @param qcdbUrl QCDB URL
-/// @param ccdbUrl CCDB URL
-/// @param outCCDBUrl URL of the CCDB where the reject list will be uploaded
-/// @return Reject list
-std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const char* qcdbUrl = "http://ali-qcdb-gpn.cern.ch:8083", const char* ccdbUrl = "http://alice-ccdb.cern.ch", const char* outCCDBUrl = "http://localhost:8080")
+/// @brief Merges the manual and automatic reject lists
+/// @param manualRL Manual reject list from json file
+/// @param rls Reject list from QCDB and CCDB
+/// @return Merged reject list
+std::vector<RejectListStruct> merge_rejectlists(const RejectListStruct& manualRL, const std::vector<RejectListStruct>& rls)
 {
-  // Get the QC quality object for the selected timestamp
-  o2::ccdb::CcdbApi qcdbApi;
-  qcdbApi.init(qcdbUrl);
-  o2::ccdb::CcdbApi ccdbApi;
-  ccdbApi.init(ccdbUrl);
-  o2::ccdb::CcdbApi outCCDBApi;
-  outCCDBApi.init(outCCDBUrl);
-  return build_rejectlist(timestamp, qcdbApi, ccdbApi, outCCDBApi);
+  std::vector<RejectListStruct> merged;
+  if (rls.empty()) {
+    merged.emplace_back(manualRL);
+    return merged;
+  }
+  o2::mid::ColumnDataHandler ch;
+  RejectListStruct tmpRL;
+  long lastEnd = manualRL.start;
+  for (auto& rl : rls) {
+    std::cout << "Checking rl with validity:      " << timeRangeToString(rl.start, rl.end) << std::endl;
+    if (rl.start >= manualRL.start && rl.end <= manualRL.end) {
+      // The period is included in the validity of the manual reject list
+      if (rl.start > lastEnd) {
+        // Fill holes between periods
+        tmpRL = manualRL;
+        tmpRL.start = lastEnd;
+        tmpRL.end = rl.start;
+        merged.emplace_back(tmpRL);
+        std::cout << "Adding manual RL with validity: " << timeRangeToString(tmpRL.start, tmpRL.end) << std::endl;
+      }
+      lastEnd = rl.end;
+
+      // merge
+      ch.clear();
+      ch.merge(rl.rejectList);
+      ch.merge(manualRL.rejectList);
+      tmpRL = rl;
+      tmpRL.rejectList = ch.getMerged();
+      std::sort(tmpRL.rejectList.begin(), tmpRL.rejectList.end(), [](const o2::mid::ColumnData& col1, const o2::mid::ColumnData& col2) { return o2::mid::getColumnDataUniqueId(col1.deId, col1.columnId) < o2::mid::getColumnDataUniqueId(col2.deId, col2.columnId); });
+      merged.emplace_back(tmpRL);
+      std::cout << "Merging RL with validity:       " << timeRangeToString(tmpRL.start, tmpRL.end) << std::endl;
+      // std::cout << "Before: " << std::endl;
+      // for (auto& col : rl.rejectList) {
+      //   std::cout << col << std::endl;
+      // }
+      // std::cout << "After: " << std::endl;
+      // for (auto& col : tmpRL.rejectList) {
+      //   std::cout << col << std::endl;
+      // }
+    } else {
+      if (rl.start > manualRL.end && lastEnd < manualRL.end) {
+        // Close manual period
+        tmpRL = manualRL;
+        tmpRL.start = lastEnd;
+        merged.emplace_back(tmpRL);
+        std::cout << "Adding manual RL with validity: " << timeRangeToString(tmpRL.start, tmpRL.end) << std::endl;
+        lastEnd = manualRL.end;
+      }
+      // Add current reject list as it is
+      merged.emplace_back(rl);
+      std::cout << "Adding RL with validity: " << timeRangeToString(rl.start, rl.end) << std::endl;
+    }
+  }
+  return merged;
 }
 
 /// @brief Builds the reject list iin a time range
@@ -282,17 +383,38 @@ std::vector<o2::mid::ColumnData> build_rejectlist(long timestamp, const char* qc
 /// @param qcdbUrl QCDB URL
 /// @param ccdbUrl CCDB URL
 /// @param outCCDBUrl URL of the CCDB where the reject lists will be uploaded
-void build_rejectlist(long start, long end, const char* qcdbUrl = "http://ali-qcdb-gpn.cern.ch:8083", const char* ccdbUrl = "http://alice-ccdb.cern.ch", const char* outCCDBUrl = "http://localhost:8080")
+void build_rejectlist(long start, long end, const char* qcdbUrl = "http://ali-qcdb-gpn.cern.ch:8083", const char* ccdbUrl = "http://alice-ccdb.cern.ch", const char* outCCDBUrl = "http://localhost:8080", const char* json_rejectlist = "")
 {
   // Query the MID QC quality objects
   o2::ccdb::CcdbApi qcdbApi;
   qcdbApi.init(qcdbUrl);
   o2::ccdb::CcdbApi ccdbApi;
   ccdbApi.init(ccdbUrl);
-  o2::ccdb::CcdbApi outCCDBApi;
-  outCCDBApi.init(outCCDBUrl);
+  std::vector<RejectListStruct> rls;
   auto objectsTS = findObjectsTSInPeriod(start, end, qcdbApi, sPathQCQuality.c_str());
   for (auto ts : objectsTS) {
-    build_rejectlist(ts, qcdbApi, ccdbApi, outCCDBApi);
+    auto rl = build_rejectlist(ts, qcdbApi, ccdbApi);
+    if (rl.start != rl.end) {
+      rls.emplace_back(rl);
+    }
+  }
+
+  if (!std::string(json_rejectlist).empty()) {
+    auto rlManual = load_from_json(ccdbApi, json_rejectlist);
+    rls = merge_rejectlists(rlManual, rls);
+  }
+
+  o2::ccdb::CcdbApi outCCDBApi;
+  outCCDBApi.init(outCCDBUrl);
+  std::map<std::string, std::string> metadata;
+  for (auto& rl : rls) {
+    // Ask if you want to upload the object to the CCDB
+    std::cout << "Upload reject list with validity: " << rl.start << " - " << rl.end << " to " << outCCDBApi.getURL() << "? [y/n]" << std::endl;
+    std::string answer;
+    std::cin >> answer;
+    if (answer == "y") {
+      std::cout << "Storing RejectList valid from " << rl.start << " to " << rl.end << std::endl;
+      outCCDBApi.storeAsTFileAny(&rl.rejectList, "MID/Calib/RejectList", metadata, rl.start, rl.end);
+    }
   }
 }