fixup! QC-1086 added cycle handling to Aggregators

Michal Tichák · Michal Tichák · commit d562e7208f0e · 2025-07-22T14:08:19.000+02:00
diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt
@@ -136,7 +136,7 @@ add_library(O2QualityControl
   src/ReductorHelpers.cxx
   src/KafkaPoller.cxx
   src/FlagHelpers.cxx
-  src/ObjectMetadataKeysHelpers.cxx)
+  src/ObjectMetadataHelpers.cxx)
 
 target_include_directories(
   O2QualityControl
diff --git a/Framework/include/QualityControl/DatabaseInterface.h b/Framework/include/QualityControl/DatabaseInterface.h
@@ -133,6 +133,7 @@ class DatabaseInterface
    * @param objectName Name of the object
    * @param timestamp Timestamp of the object in ms since epoch
    * @param activity Activity of the object
+   * @param metadata additional metadata to filter objects during retrieval
    * @deprecated
    */
   virtual std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string objectPath, std::string objectName,
@@ -144,6 +145,7 @@ class DatabaseInterface
    * @param qoPath Path of the object without the provenance prefix
    * @param timestamp Timestamp of the object in ms since epoch
    * @param activity Activity of the object
+   * @param metadata additional metadata to filter objects during retrieval
    * @deprecated
    */
   virtual std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = Timestamp::Current,
diff --git a/Framework/include/QualityControl/ObjectMetadataHelpers.h b/Framework/include/QualityControl/ObjectMetadataHelpers.h
@@ -10,14 +10,17 @@
 // or submit itself to any jurisdiction.
 
 ///
-/// \file   ObjectMetadataKeysHelpers.h
+/// \file   ObjectMetadataHelpers.h
 /// \author Michal Tichak
 ///
 
+#ifndef QUALITYCONTROL_OBJECTMETADATAHELPERS_H
+#define QUALITYCONTROL_OBJECTMETADATAHELPERS_H
+
 #include <optional>
 #include <string>
 
-namespace o2::quality_control::repository::metadata_keys
+namespace o2::quality_control::repository
 {
 /**
  * \brief Parses metadata value stored under metadata_keys::cycle
@@ -26,4 +29,6 @@ namespace o2::quality_control::repository::metadata_keys
  *
  */
 std::optional<unsigned long> parseCycle(const std::string& cycleStr);
-} // namespace o2::quality_control::repository::metadata_keys
+} // namespace o2::quality_control::repository
+
+#endif
diff --git a/Framework/include/QualityControl/ObjectMetadataKeys.h b/Framework/include/QualityControl/ObjectMetadataKeys.h
@@ -37,7 +37,7 @@ constexpr auto qcTaskClass = "qc_task_class";
 constexpr auto qcQuality = "qc_quality";
 constexpr auto qcCheckName = "qc_check_name";
 constexpr auto qcAdjustableEOV = "adjustableEOV"; // this is a keyword for the CCDB
-constexpr auto cycle = "CycleNumber";
+constexpr auto cycleNumber = "CycleNumber";
 
 // QC Activity
 constexpr auto runType = "RunType";
diff --git a/Framework/src/Aggregator.cxx b/Framework/src/Aggregator.cxx
@@ -20,7 +20,7 @@
 #include "QualityControl/QualityObject.h"
 #include "QualityControl/RootClassFactory.h"
 #include "QualityControl/AggregatorInterface.h"
-#include "QualityControl/ObjectMetadataKeysHelpers.h"
+#include "QualityControl/ObjectMetadataHelpers.h"
 #include "QualityControl/UpdatePolicyType.h"
 #include "QualityControl/ActivityHelpers.h"
 #include "QualityControl/Activity.h"
@@ -113,9 +113,9 @@ std::optional<unsigned long> getMaxCycle(const QualityObjectsMapType& qoMap)
 {
   std::optional<unsigned long> max{};
   for (const auto& [_, qo] : qoMap) {
-    auto cycle = qo->getMetadataOpt(repository::metadata_keys::cycle);
+    auto cycle = qo->getMetadataOpt(repository::metadata_keys::cycleNumber);
     if (cycle.has_value()) {
-      auto parsedCycle = repository::metadata_keys::parseCycle(cycle.value());
+      auto parsedCycle = repository::parseCycle(cycle.value());
       if (parsedCycle) {
         max = std::max(parsedCycle.value(), max.value_or(0));
       }
@@ -162,7 +162,7 @@ QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Act
       UpdatePolicyTypeUtils::ToString(mAggregatorConfig.policyType)));
     qualityObjects.back()->setActivity(resultActivity);
     if (maxCycle.has_value()) {
-      qualityObjects.back()->addMetadata(repository::metadata_keys::cycle, std::to_string(maxCycle.value()));
+      qualityObjects.back()->addMetadata(repository::metadata_keys::cycleNumber, std::to_string(maxCycle.value()));
     }
   }
   return qualityObjects;
diff --git a/Framework/src/Check.cxx b/Framework/src/Check.cxx
@@ -30,7 +30,7 @@
 #include "QualityControl/QcInfoLogger.h"
 #include "QualityControl/Quality.h"
 #include "QualityControl/HashDataDescription.h"
-#include "QualityControl/ObjectMetadataKeysHelpers.h"
+#include "QualityControl/ObjectMetadataHelpers.h"
 
 #include <QualityControl/AggregatorRunner.h>
 
@@ -168,9 +168,9 @@ QualityObjectsType Check::check(std::map<std::string, std::shared_ptr<MonitorObj
     std::optional<unsigned long> maxCycle{};
     for (const auto& [moName, mo] : moMapToCheck) {
       monitorObjectsNames.emplace_back(moName);
-      if (const auto cycle = mo->getMetadata(repository::metadata_keys::cycle)) {
+      if (const auto cycle = mo->getMetadata(repository::metadata_keys::cycleNumber)) {
         const auto& cycleStr = cycle.value();
-        if (const auto cycleVal = repository::metadata_keys::parseCycle(cycleStr); cycleVal.has_value()) {
+        if (const auto cycleVal = repository::parseCycle(cycleStr); cycleVal.has_value()) {
           maxCycle = std::max(cycleVal.value(), maxCycle.value_or(0));
         }
       }
@@ -186,7 +186,7 @@ QualityObjectsType Check::check(std::map<std::string, std::shared_ptr<MonitorObj
 
     qualityObjects.back()->setActivity(commonActivity);
     if (maxCycle.has_value()) {
-      qualityObjects.back()->addMetadata(repository::metadata_keys::cycle, std::to_string(maxCycle.value()));
+      qualityObjects.back()->addMetadata(repository::metadata_keys::cycleNumber, std::to_string(maxCycle.value()));
     }
     beautify(moMapToCheck, quality);
   }
diff --git a/Framework/src/MonitorObjectCollection.cxx b/Framework/src/MonitorObjectCollection.cxx
@@ -18,7 +18,7 @@
 #include "QualityControl/MonitorObject.h"
 #include "QualityControl/ObjectMetadataKeys.h"
 #include "QualityControl/QcInfoLogger.h"
-#include "QualityControl/ObjectMetadataKeysHelpers.h"
+#include "QualityControl/ObjectMetadataHelpers.h"
 
 #include <Mergers/MergerAlgorithm.h>
 #include <TNamed.h>
@@ -32,24 +32,24 @@ namespace o2::quality_control::core
 
 void mergeCycles(MonitorObject* targetMO, MonitorObject* otherMO)
 {
-  const auto otherCycle = otherMO->getMetadata(repository::metadata_keys::cycle);
-  const auto targetCycle = targetMO->getMetadata(repository::metadata_keys::cycle);
+  const auto otherCycle = otherMO->getMetadata(repository::metadata_keys::cycleNumber);
+  const auto targetCycle = targetMO->getMetadata(repository::metadata_keys::cycleNumber);
   if (otherCycle.has_value() && targetCycle.has_value()) {
-    const auto targetCycleParsed = repository::metadata_keys::parseCycle(targetCycle.value());
-    const auto otherCycleParsed = repository::metadata_keys::parseCycle(otherCycle.value());
+    const auto targetCycleParsed = repository::parseCycle(targetCycle.value());
+    const auto otherCycleParsed = repository::parseCycle(otherCycle.value());
 
     if (targetCycleParsed && otherCycleParsed) {
-      targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(std::max(targetCycleParsed.value(), otherCycleParsed.value())));
+      targetMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(std::max(targetCycleParsed.value(), otherCycleParsed.value())));
       return;
     }
 
     if (targetCycleParsed.value()) {
-      targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(targetCycleParsed.value()));
+      targetMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(targetCycleParsed.value()));
       return;
     }
 
     if (otherCycleParsed.value()) {
-      otherMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(otherCycleParsed.value()));
+      otherMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(otherCycleParsed.value()));
       return;
     }
   }
diff --git a/Framework/src/ObjectMetadataHelpers.cxx b/Framework/src/ObjectMetadataHelpers.cxx
@@ -10,26 +10,26 @@
 // or submit itself to any jurisdiction.
 
 ///
-/// \file   ObjectMetadataKeysHelpers.cxx
+/// \file   ObjectMetadataHelpers.cxx
 /// \author Michal Tichak
 ///
 
 #include <charconv>
-#include "QualityControl/ObjectMetadataKeysHelpers.h"
+#include "QualityControl/ObjectMetadataHelpers.h"
 #include "QualityControl/ObjectMetadataKeys.h"
 #include "QualityControl/QcInfoLogger.h"
 
-namespace o2::quality_control::repository::metadata_keys
+namespace o2::quality_control::repository
 {
 std::optional<unsigned long> parseCycle(const std::string& cycleStr)
 {
   unsigned long cycleVal{};
   if (auto parse_res = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal);
       parse_res.ec != std::errc{}) {
-    ILOG(Warning, Support) << "failed to decypher " << repository::metadata_keys::cycle << " metadata with value " << cycleStr
+    ILOG(Warning, Support) << "failed to decypher " << repository::metadata_keys::cycleNumber << " metadata with value " << cycleStr
                            << ", with std::errc " << std::make_error_code(parse_res.ec).message() << ENDM;
     return std::nullopt;
   }
   return cycleVal;
 }
-} // namespace o2::quality_control::repository::metadata_keys
+} // namespace o2::quality_control::repository
diff --git a/Framework/src/TaskRunner.cxx b/Framework/src/TaskRunner.cxx
@@ -524,7 +524,7 @@ int TaskRunner::publish(DataAllocator& outputs)
   // getNonOwningArray creates a TObjArray containing the monitoring objects, but not
   // owning them. The array is created by new and must be cleaned up by the caller
   std::unique_ptr<MonitorObjectCollection> array(mObjectsManager->getNonOwningArray());
-  array->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(mCycleNumber));
+  array->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(mCycleNumber));
   int objectsPublished = array->GetEntries();
 
   outputs.snapshot(
diff --git a/Framework/src/Triggers.cxx b/Framework/src/Triggers.cxx
@@ -293,8 +293,8 @@ TriggerFcn ForEachObject(const std::string& databaseUrl, const std::string& data
       auto currentActivity = activity_helpers::asActivity(*currentObject, activity.mProvenance);
       bool last = currentObject + 1 == filteredObjects->end();
       Trigger trigger(TriggerType::ForEachObject, last, currentActivity, currentObject->get<int64_t>(timestampSortKey));
-      if (auto cycle = currentObject->get_optional<std::string>(metadata_keys::cycle); cycle.has_value()) {
-        trigger.metadata.emplace(metadata_keys::cycle, cycle.value());
+      if (auto cycle = currentObject->get_optional<std::string>(metadata_keys::cycleNumber); cycle.has_value()) {
+        trigger.metadata.emplace(metadata_keys::cycleNumber, cycle.value());
       }
       ++currentObject;
 
diff --git a/Framework/test/testAggregatorRunner.cxx b/Framework/test/testAggregatorRunner.cxx
@@ -217,13 +217,13 @@ TEST_CASE("test_aggregator_cycle")
   QualityObjectsMapType qoMap;
   auto qo1 = make_shared<QualityObject>(Quality::Good, "checkAll");
   qoMap["checkAll"] = qo1;
-  qo1->addMetadata(o2::quality_control::repository::metadata_keys::cycle, "1");
+  qo1->addMetadata(o2::quality_control::repository::metadata_keys::cycleNumber, "1");
   auto qo2 = make_shared<QualityObject>(Quality::Bad, "dataSizeCheck2/skeletonTask/example");
-  qo2->addMetadata(o2::quality_control::repository::metadata_keys::cycle, "2");
+  qo2->addMetadata(o2::quality_control::repository::metadata_keys::cycleNumber, "2");
   qoMap["dataSizeCheck2/skeletonTask/example"] = qo2;
   auto results = aggregator->aggregate(qoMap);
   for (auto r : results) {
-    REQUIRE_NOTHROW(r->getMetadata(o2::quality_control::repository::metadata_keys::cycle));
-    CHECK(r->getMetadata(o2::quality_control::repository::metadata_keys::cycle) == "2");
+    REQUIRE_NOTHROW(r->getMetadata(o2::quality_control::repository::metadata_keys::cycleNumber));
+    CHECK(r->getMetadata(o2::quality_control::repository::metadata_keys::cycleNumber) == "2");
   }
 }
diff --git a/Framework/test/testCcdbDatabase.cxx b/Framework/test/testCcdbDatabase.cxx
@@ -133,13 +133,13 @@ BOOST_AUTO_TEST_CASE(ccdb_store)
 
   TH1F* h5 = new TH1F("cycle", "asdf", 100, 0, 99);
   shared_ptr<MonitorObject> mo5 = make_shared<MonitorObject>(h5, f.taskName, "TestClass", "TST");
-  mo5->addMetadata(metadata_keys::cycle, "1");
+  mo5->addMetadata(metadata_keys::cycleNumber, "1");
   mo5->setValidity({ 10000, 20000 });
   mo5->updateActivity(1234, "LHC66", "passName1", "qc");
 
   TH1F* h6 = new TH1F("cycle", "asdf", 100, 0, 99);
   shared_ptr<MonitorObject> mo6 = make_shared<MonitorObject>(h6, f.taskName, "TestClass", "TST");
-  mo6->addMetadata(metadata_keys::cycle, "2");
+  mo6->addMetadata(metadata_keys::cycleNumber, "2");
   mo6->setValidity({ 10000, 20000 });
   mo6->updateActivity(1234, "LHC66", "passName1", "qc");
 
@@ -232,16 +232,16 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve_mo_with_cycle, *utf::depends_on("ccdb_store")
   test_fixture f;
   std::shared_ptr<MonitorObject> mo{};
   mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle",
-                             15000, Activity{}, { { metadata_keys::cycle, "1" } });
+                             15000, Activity{}, { { metadata_keys::cycleNumber, "1" } });
   BOOST_REQUIRE(mo.get() != nullptr);
-  BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle));
-  BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "1");
+  BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycleNumber));
+  BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycleNumber) == "1");
 
   mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle",
-                             15000, Activity{}, { { metadata_keys::cycle, "2" } });
+                             15000, Activity{}, { { metadata_keys::cycleNumber, "2" } });
   BOOST_REQUIRE(mo.get() != nullptr);
-  BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle));
-  BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "2");
+  BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycleNumber));
+  BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycleNumber) == "2");
 }
 
 BOOST_AUTO_TEST_CASE(ccdb_retrieve_qo, *utf::depends_on("ccdb_store"))
diff --git a/Framework/test/testMonitorObjectCollection.cxx b/Framework/test/testMonitorObjectCollection.cxx
@@ -257,19 +257,19 @@ TEST_CASE("monitor_object_collection_merge_cycle")
   MonitorObject* targetMoTH1I = new MonitorObject(targetTH1I, "histo 1d", "class", "DET");
   targetMoTH1I->setIsOwner(true);
   target.Add(targetMoTH1I);
-  targetMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycle, "1");
+  targetMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, "1");
 
   other.SetOwner(true);
 
   TH1I* otherTH1I = new TH1I("histo 1d", "histo 1d", bins, min, max);
   MonitorObject* otherMoTH1I = new MonitorObject(otherTH1I, "histo 1d", "class", "DET");
   otherMoTH1I->setIsOwner(true);
   other.Add(otherMoTH1I);
-  otherMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycle, "2");
+  otherMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, "2");
 
   algorithm::merge(&target, &other);
 
-  const auto mergedCycle = targetMoTH1I->getMetadata(repository::metadata_keys::cycle);
+  const auto mergedCycle = targetMoTH1I->getMetadata(repository::metadata_keys::cycleNumber);
   REQUIRE(mergedCycle.has_value());
   REQUIRE(mergedCycle.value() == "2");
 }
diff --git a/Framework/test/testTriggers.cxx b/Framework/test/testTriggers.cxx
@@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object)
   mo->setActivity({ 101, "TECHNICAL", "FCC42x", "tpass1", "qc", { currentTimestamp + 1000, gInvalidValidityInterval.getMax() } });
   repository->storeMO(mo);
   mo->setActivity({ 100, "TECHNICAL", "FCC42x", "tpass2", "qc", { currentTimestamp + 2000, gInvalidValidityInterval.getMax() } });
-  mo->addOrUpdateMetadata(metadata_keys::cycle, "42");
+  mo->addOrUpdateMetadata(metadata_keys::cycleNumber, "42");
   repository->storeMO(mo);
 
   {
@@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object)
     trigger = forEachObjectTrigger();
     BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject);
     BOOST_CHECK_EQUAL(trigger.metadata.size(), 1);
-    BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycle), "42");
+    BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycleNumber), "42");
 
     BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::No);
   }
@@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object)
     trigger = forEachObjectTrigger();
     BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject);
     BOOST_CHECK_EQUAL(trigger.metadata.size(), 1);
-    BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycle), "42");
+    BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycleNumber), "42");
 
     BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::No);
   }
diff --git a/doc/Framework.md b/doc/Framework.md
@@ -458,6 +458,7 @@ export O2_DPL_DEPLOYMENT_MODE=Grid && o2-qc --local-batch QC.root ...
 ## Monitor cycles
 
 The QC tasks monitor and process data continuously during a so-called "monitor cycle". At the end of such a cycle they publish the QC objects that will then continue their way in the QC data flow.
+Since v1.178.0 cycles are stored in the metadata of QOs and MOs under the key `CycleNumber` inside the QCDB.
 
 A monitor cycle lasts typically between **1 and 5 minutes**, some reaching 10 minutes but never less than 1 minute for performance reasons.
 It is defined in the config file this way:
diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md
@@ -154,7 +154,7 @@ Each of the three methods can be invoked by one or more triggers. Below are list
 * `"eof"` or `"endoffill"` - End Of Fill (not implemented yet)
 * `"<x><sec/min/hour>"` - Periodic - triggers when a specified period of time passes. For example: "5min", "0.001 seconds", "10sec", "2hours".
 * `"newobject:[qcdb/ccdb]:<path>"` - New Object - triggers when an object in QCDB or CCDB is updated (applicable for synchronous processing). For example: `"newobject:qcdb:qc/TST/MO/QcTask/Example"`
-* `"foreachobject:[qcdb/ccdb]:<path>"` - For Each Object - triggers for each object in QCDB or CCDB which matches the activity indicated in the QC config file (applicable for asynchronous processing).
+* `"foreachobject:[qcdb/ccdb]:<path>"` - For Each Object - triggers for each object in QCDB or CCDB which matches the activity indicated in the QC config file (applicable for both synchronous and asynchronous processing). This trigger contains monitor cycle of required object in its metadata since v1.178.0
 * `"foreachlatest:[qcdb/ccdb]:<path>"` - For Each Latest - triggers for the latest object version in QCDB or CCDB
    for each matching activity (applicable for asynchronous processing). It sorts objects in ascending order by period,
    pass and run.
@@ -1054,7 +1054,7 @@ For TrendingTask, it would be:
     ]
 ```
 
-### I want to run postprocessing on all already existing objects for a run
+### I want to run postprocessing on all already existing objects for a run (usable in sync AND async)
 
 Use ForEachObject as the update trigger:
 
@@ -1063,7 +1063,7 @@ Use ForEachObject as the update trigger:
 ```
 
 Since objects are usually published in collections at the same time, you can use a path for one object to be triggered
- for a collection of them (all objects produced by a QC Task).
+ for a collection of them (all objects produced by a QC Task). 
 
 Use the Activity which matches the run, and (optionally) period and pass name:
 
