add possibility to update existing objects (#13341)

Author: pillot

Detectors/MUON/MCH/Conditions/README.md

@@ -30,7 +30,11 @@ Usage:
                                         (default=now)
   --endtimestamp arg (=1677773918645)   end of validity (for put) - default=1
                                         day from now
-  -p [ --put ]                          upload bad channel default object
+  -l [ --list ]                         list timestamps, within the given
+                                        range, when the bad channels change
+  -p [ --put ]                          upload bad channel object
+  -r [ --referenceccdb ] arg (=http://alice-ccdb.cern.ch)
+                                        reference ccdb url
   -u [ --upload-default-values ]        upload default values
   -t [ --type ] arg (=BadChannel)       type of bad channel (BadChannel or
                                         RejectList)
@@ -40,12 +44,12 @@ Usage:
   -d [ --ds ] arg                       dual sampas indices to reject
   -e [ --de ] arg                       DE ids to reject
   -a [ --alias ] arg                    DCS alias (HV or LV) to reject
-
 ```
 
-For instance, to create a debug RejectList object which declares solar number 32 as bad within a local CCDB, from Tuesday 1 November 2022 00:00:01 UTC to Saturday 31 December 2022 23:59:59, use :
+For instance, to create in a local CCDB a RejectList object which declares solar number 32 as bad, from Tuesday 1 November 2022 00:00:01 UTC to Saturday 31 December 2022 23:59:59, use :
 
 ```shell
 $ o2-mch-bad-channels-ccdb -p -s 32 -t RejectList --starttimestamp 1667260801000 --endtimestamp 1672531199000
-storing default MCH bad channels (valid from 1667260801000 to 1672531199000) to MCH/Calib/RejectList
 ```
+
+The program will search the reference CCDB (defined with `--referenceccdb`) for existing objects valid during this period and propose you to either overwrite them or update them. In the first case, a single object will be created, valid for the whole period, containing only the new bad channels. In the second case, as many objects as necessary will be created with appropriate validity ranges, adding the new bad channels to the existing ones.

Detectors/MUON/MCH/Conditions/src/bad-channels-ccdb.cxx

@@ -9,13 +9,16 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include "CCDB/CcdbApi.h"
 #include <boost/program_options.hpp>
 #include <ctime>
+#include <fstream>
+#include <iterator>
 #include <numeric>
 #include <regex>
+#include <set>
 #include <string>
 #include <vector>
+#include "CCDB/CcdbApi.h"
 #include "DataFormatsMCH/DsChannelId.h"
 #include "MCHConditions/DCSAliases.h"
 #include "MCHConstants/DetectionElements.h"
@@ -33,20 +36,105 @@ std::string ccdbPath(const std::string badChannelType)
   return fmt::format("MCH/Calib/{}", badChannelType);
 }
 
-void queryBadChannels(const std::string ccdbUrl,
-                      const std::string badChannelType, uint64_t timestamp, bool verbose)
+std::set<uint64_t> listTSWhenBadChannelsChange(const std::string ccdbUrl, const std::string badChannelType,
+                                               uint64_t startTimestamp, uint64_t endTimestamp, bool verbose)
+{
+  std::set<uint64_t> tsChanges{startTimestamp};
+
+  std::cout << std::endl;
+  o2::ccdb::CcdbApi api;
+  api.init(ccdbUrl);
+  auto source = ccdbPath(badChannelType);
+
+  // store every timestamps in the time range when the bad channels potentially change
+  if (verbose) {
+    std::cout << fmt::format("\nlist of {} files potentially valid in the range {} - {}:\n", source, startTimestamp, endTimestamp);
+  }
+  std::istringstream fileInfo(api.list(source, false, "text/plain"));
+  std::string dummy{};
+  std::string path{};
+  uint64_t begin = 0;
+  uint64_t end = 0;
+  uint64_t creation = 0;
+  bool inRange = false;
+  for (std::string line; std::getline(fileInfo, line);) {
+    if (line.find("Validity:") == 0) {
+      std::istringstream in(line);
+      in >> dummy >> begin >> dummy >> end;
+      if (begin < endTimestamp && end > startTimestamp) {
+        if (begin >= startTimestamp) {
+          tsChanges.emplace(begin);
+        }
+        if (end < endTimestamp) {
+          tsChanges.emplace(end);
+        }
+        inRange = true;
+      }
+    } else if (verbose) {
+      if (line.find("ID:") == 0) {
+        std::istringstream in(line);
+        in >> dummy >> path;
+      } else if (inRange && line.find("Created:") == 0) {
+        std::istringstream in(line);
+        in >> dummy >> creation;
+        std::cout << fmt::format("- {}\n", path);
+        std::cout << fmt::format("  validity range: {} - {}\n", begin, end);
+        std::cout << fmt::format("  creation time: {}\n", creation);
+        inRange = false;
+      }
+    }
+  }
+  if (verbose) {
+    std::cout << fmt::format("\nlist of timestamps when the bad channels potentially change:\n");
+    for (auto ts : tsChanges) {
+      std::cout << fmt::format("  {}\n", ts);
+    }
+  }
+
+  // select timestamps when the bad channels actually change
+  if (verbose) {
+    std::cout << fmt::format("\nlist of {} files actually valid in the range {} - {}:\n", source, startTimestamp, endTimestamp);
+  }
+  std::map<std::string, std::string> metadata{};
+  std::string currentETag{};
+  for (auto itTS = tsChanges.begin(); itTS != tsChanges.end();) {
+    auto headers = api.retrieveHeaders(source, metadata, *itTS);
+    if (headers["ETag"] == currentETag) {
+      itTS = tsChanges.erase(itTS);
+    } else {
+      if (verbose) {
+        std::cout << fmt::format("- {}\n", headers["Location"]);
+        std::cout << fmt::format("  validity range: {} - {}\n", headers["Valid-From"], headers["Valid-Until"]);
+        std::cout << fmt::format("  creation time: {}\n", headers["Created"]);
+      }
+      currentETag = headers["ETag"];
+      ++itTS;
+    }
+  }
+  std::cout << fmt::format("\nlist of timestamps when the bad channels actually change:\n");
+  for (auto ts : tsChanges) {
+    std::cout << fmt::format("  {}\n", ts);
+  }
+
+  return tsChanges;
+}
+
+BadChannelsVector queryBadChannels(const std::string ccdbUrl,
+                                   const std::string badChannelType, uint64_t timestamp, bool verbose)
 {
+  std::cout << std::endl;
   o2::ccdb::CcdbApi api;
   api.init(ccdbUrl);
   std::map<std::string, std::string> metadata;
   auto source = ccdbPath(badChannelType);
-  auto* badChannels = api.retrieveFromTFileAny<BadChannelsVector>(source.c_str(), metadata, timestamp);
+  auto* badChannels = api.retrieveFromTFileAny<BadChannelsVector>(source, metadata, timestamp);
   std::cout << "number of bad channels = " << badChannels->size() << std::endl;
   if (verbose) {
     for (const auto& badChannel : *badChannels) {
       std::cout << badChannel.asString() << "\n";
     }
   }
+  return *badChannels;
 }
 
 void rejectDS(const o2::mch::raw::DsDetId& dsDetId, BadChannelsVector& bv)
@@ -124,13 +212,13 @@ void uploadBadChannels(const std::string ccdbUrl,
                        const BadChannelsVector& bv,
                        bool makeDefault)
 {
+  std::cout << std::endl;
   o2::ccdb::CcdbApi api;
   api.init(ccdbUrl);
   std::map<std::string, std::string> md;
   auto dest = ccdbPath(badChannelType);
-  std::cout << "storing default MCH bad channels (valid from "
-            << startTimestamp << " to " << endTimestamp << ") to "
-            << dest << "\n";
+  std::cout << fmt::format("storing {} {}bad channels (valid from {} to {}) to {}\n", bv.size(),
+                           makeDefault ? "default " : "", startTimestamp, endTimestamp, dest);
 
   if (makeDefault) {
     md["default"] = "true";
@@ -145,10 +233,12 @@ int main(int argc, char** argv)
   po::options_description usage("Usage");
 
   std::string ccdbUrl;
+  std::string ccdbRefUrl;
   std::string dpConfName;
   std::string badChannelType;
   uint64_t startTimestamp;
   uint64_t endTimestamp;
+  bool list;
   bool put;
   bool query;
   bool verbose;
@@ -165,9 +255,11 @@ int main(int argc, char** argv)
   usage.add_options()
       ("help,h", "produce help message")
       ("ccdb,c",po::value<std::string>(&ccdbUrl)->default_value("http://localhost:6464"),"ccdb url")
-      ("starttimestamp,st",po::value<uint64_t>(&startTimestamp)->default_value(now),"timestamp for query or put - (default=now)")
-      ("endtimestamp,et", po::value<uint64_t>(&endTimestamp)->default_value(end), "end of validity (for put) - default=1 day from now")
-      ("put,p",po::bool_switch(&put),"upload bad channel default object")
+      ("starttimestamp",po::value<uint64_t>(&startTimestamp)->default_value(now),"timestamp for query or put - (default=now)")
+      ("endtimestamp", po::value<uint64_t>(&endTimestamp)->default_value(end), "end of validity (for put) - default=1 day from now")
+      ("list,l", po::bool_switch(&list),"list timestamps, within the given range, when the bad channels change")
+      ("put,p",po::bool_switch(&put),"upload bad channel object")
+      ("referenceccdb,r",po::value<std::string>(&ccdbRefUrl)->default_value("http://alice-ccdb.cern.ch"),"reference ccdb url")
       ("upload-default-values,u",po::bool_switch(&uploadDefault),"upload default values")
       ("type,t",po::value<std::string>(&badChannelType)->default_value("BadChannel"),"type of bad channel (BadChannel or RejectList)")
       ("query,q",po::bool_switch(&query),"dump bad channel object from CCDB")
@@ -202,6 +294,10 @@ int main(int argc, char** argv)
     exit(2);
   }
 
+  if (list) {
+    auto tsChanges = listTSWhenBadChannelsChange(ccdbUrl, badChannelType, startTimestamp, endTimestamp, verbose);
+  }
+
   if (query) {
     queryBadChannels(ccdbUrl, badChannelType, startTimestamp, verbose);
   }
@@ -221,7 +317,39 @@ int main(int argc, char** argv)
       rejectHVLVs(vm["alias"].as<std::vector<std::string>>(), bv);
     }
 
-    uploadBadChannels(ccdbUrl, badChannelType, startTimestamp, endTimestamp, bv, uploadDefault);
+    if (uploadDefault) {
+      uploadBadChannels(ccdbUrl, badChannelType, startTimestamp, endTimestamp, bv, true);
+    } else {
+      auto tsChanges = listTSWhenBadChannelsChange(ccdbRefUrl, badChannelType, startTimestamp, endTimestamp, false);
+
+      std::cout << fmt::format("\n{} object{} valid in the reference CCDB ({}) for this time range. What do you want to do?\n",
+                               tsChanges.size(), (tsChanges.size() > 1) ? "s are" : " is", ccdbRefUrl);
+      std::cout << fmt::format("[a] abort: do nothing\n");
+      std::cout << fmt::format("[o] overwrite: create 1 new object for the whole time range to supersede the existing one{}\n",
+                               (tsChanges.size() > 1) ? "s" : "");
+      std::cout << fmt::format("[u] update: create {} new object{} within the time range adding new bad channels to existing ones\n",
+                               tsChanges.size(), (tsChanges.size() > 1) ? "s" : "");
+      std::string response{};
+      std::cin >> response;
+
+      if (response == "a" || response == "abort") {
+        return 0;
+      } else if (response == "o" || response == "overwrite") {
+        uploadBadChannels(ccdbUrl, badChannelType, startTimestamp, endTimestamp, bv, false);
+      } else if (response == "u" || response == "update") {
+        tsChanges.emplace(endTimestamp);
+        auto itStartTS = tsChanges.begin();
+        for (auto itStopTS = std::next(itStartTS); itStopTS != tsChanges.end(); ++itStartTS, ++itStopTS) {
+          auto bv2 = queryBadChannels(ccdbRefUrl, badChannelType, *itStartTS, false);
+          bv2.insert(bv2.end(), bv.begin(), bv.end());
+          uploadBadChannels(ccdbUrl, badChannelType, *itStartTS, *itStopTS, bv2, false);
+        }
+      } else {
+        std::cout << "invalid response (must be a, o or u) --> abort\n";
+        exit(3);
+      }
+    }
   }
+
   return 0;
 }