[EMCAL-565, EMCAL-566] Prevent saving/overwriting calib multiple times at EOR (#13045)

- At the EOR the state of the calibration is saved in a root file, to be loaded in the next run
- However, in the online case, several EOR signals reach the calibration after the initial EOR
- This leads to the original root file being overwritten by another one, containing only a few (~1-20) events
- Now, 2 checks are in place that should circumvent that the calibration files are overwritten:
The variable mRunStopRequested was introduced in the CalibSpec which is set to true after the initial EOR signal, preventing further data processing
A check on the minimum number of events for the calibration (minNEventsSaveSlot, configurable in the calibParams) that only allows saving the calibration file if a reasonable statistics is reached

Co-authored-by: jokonig <jokonig@cern.ch>

Author: jokonig

Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibParams.h

@@ -87,6 +87,7 @@ struct EMCALCalibParams : public o2::conf::ConfigurableParamHelper<EMCALCalibPar
   bool requireSameFill = false;                        ///< if loading calib objects from previous run, require it to be in the same fill as the current one
   bool requireSameRunType = true;                      ///< if loading calib objects from previous run, require it to be the same run type
   int tsDiffMax = 48;                                  ///< if loading calib objects from previous run, limit time between the object being stored and loaded again (in hours)
+  unsigned int minNEventsSaveSlot = 100000;            ///< minimum amount a slot has to have in order to be taken into accoutn in finalize slot. THis is also relevant if the slot should be saved at the EOR
 
   // Parameters for pedestal calibration
   short maxPedestalRMS = 10; ///< Maximum value for RMS for pedestals (has to be tuned)

Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelCalibrator.h

@@ -167,6 +167,11 @@ void EMCALChannelCalibrator<DataInput, DataOutput>::finalizeSlot(o2::calibration
   // Extract results for the single slot
   DataInput* c = slot.getContainer();
   LOG(info) << "Finalize slot " << slot.getTFStart() << " <= TF <= " << slot.getTFEnd();
+  // check if slot contains a minimum amount of data
+  if (c->getNEvents() < EMCALCalibParams::Instance().minNEventsSaveSlot) {
+    LOG(info) << "Slot only contains " << c->getNEvents() << " events. Not saving this slot. " << EMCALCalibParams::Instance().minNEventsSaveSlot << " required";
+    return;
+  }
 
   if constexpr (std::is_same<DataInput, o2::emcal::EMCALChannelData>::value) {
     if (c->getNEvents() < EMCALCalibParams::Instance().minNEvents_bc) {
@@ -260,7 +265,7 @@ template <typename DataInput, typename DataOutput>
 bool EMCALChannelCalibrator<DataInput, DataOutput>::saveLastSlotData(TFile& fl)
 {
   LOG(info) << "EMC calib histos are saved in " << fl.GetName();
-  // does this work?
+  // we only have 1 slot
   auto& cont = o2::calibration::TimeSlotCalibration<DataInput>::getSlots();
   auto& slot = cont.at(0);
   DataInput* c = slot.getContainer();
@@ -279,7 +284,7 @@ bool EMCALChannelCalibrator<DataInput, DataOutput>::saveLastSlotData(TFile& fl)
     TH1I hGlobalProperties("hGlobalProperties", "hGlobalProperties", 3, -0.5, 2.5);
     hGlobalProperties.GetXaxis()->SetBinLabel(1, "Fill nr.");
     hGlobalProperties.GetXaxis()->SetBinLabel(2, "run type");
-    hGlobalProperties.GetXaxis()->SetBinLabel(2, "ts in hours");
+    hGlobalProperties.GetXaxis()->SetBinLabel(3, "ts in hours");
     hGlobalProperties.SetBinContent(1, mFillNr);
     hGlobalProperties.SetBinContent(2, mRunType);
     hGlobalProperties.SetBinContent(3, timeNow);
@@ -297,7 +302,7 @@ bool EMCALChannelCalibrator<DataInput, DataOutput>::saveLastSlotData(TFile& fl)
     TH1I hGlobalProperties("hGlobalProperties", "hGlobalProperties", 3, -0.5, 2.5);
     hGlobalProperties.GetXaxis()->SetBinLabel(1, "Fill nr.");
     hGlobalProperties.GetXaxis()->SetBinLabel(2, "run type");
-    hGlobalProperties.GetXaxis()->SetBinLabel(2, "ts in hours");
+    hGlobalProperties.GetXaxis()->SetBinLabel(3, "ts in hours");
     hGlobalProperties.SetBinContent(1, mFillNr);
     hGlobalProperties.SetBinContent(2, mRunType);
     hGlobalProperties.SetBinContent(3, timeNow);

Detectors/EMCAL/calibration/testWorkflow/EMCALChannelCalibratorSpec.h

@@ -172,6 +172,14 @@ class EMCALChannelCalibDevice : public o2::framework::Task
       timeMeas[0] = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now().time_since_epoch()).count();
     }
 
+    const auto& tinfo = pc.services().get<o2::framework::TimingInfo>();
+    if (tinfo.globalRunNumberChanged) { // new run is starting
+      mRunStopRequested = false;
+    }
+    if (mRunStopRequested) {
+      return;
+    }
+
     o2::base::GRPGeomHelper::instance().checkUpdates(pc);
     if (mTimeCalibrator) {
       o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mTimeCalibrator->getCurrentTFInfo());
@@ -314,7 +322,7 @@ class EMCALChannelCalibDevice : public o2::framework::Task
 
     if (pc.transitionState() == TransitionHandlingState::Requested) {
       LOG(debug) << "Run stop requested, finalizing";
-      // mRunStopRequested = true;
+      mRunStopRequested = true;
       if (isBadChannelCalib) {
         mBadChannelCalibrator->setSaveAtEOR(true);
         mBadChannelCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
@@ -337,6 +345,9 @@ class EMCALChannelCalibDevice : public o2::framework::Task
 
   void endOfStream(o2::framework::EndOfStreamContext& ec) final
   {
+    if (mRunStopRequested) {
+      return;
+    }
     if (isBadChannelCalib) {
       mBadChannelCalibrator->setSaveAtEOR(true);
       mBadChannelCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
@@ -346,6 +357,7 @@ class EMCALChannelCalibDevice : public o2::framework::Task
       mTimeCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
       sendOutput<o2::emcal::TimeCalibrationParams>(ec.outputs());
     }
+    mRunStopRequested = true;
   }
 
   static const char* getCellBinding() { return "EMCCells"; }
@@ -368,6 +380,7 @@ class EMCALChannelCalibDevice : public o2::framework::Task
   bool mRejectL0Triggers = true;                                                                                                       ///! reject EMCal Gamma and Jet triggers in the online calibration
   bool mApplyGainCalib = true;                                                                                                         ///! switch if gain calibration should be applied during filling of histograms or not
   bool mGainCalibFactorsInitialized = false;                                                                                           ///! Gain calibration init status
+  bool mRunStopRequested = false;                                                                                                      ///< flag that the run has stopped
   std::array<double, 2> timeMeas;                                                                                                      ///! Used for time measurement and holds the start and end time in the run function
   std::vector<uint64_t> mSelectedClassMasks = {};                                                                                      ///! EMCal minimum bias trigger bit. Only this bit will be used for calibration
   std::unique_ptr<CalibInputDownsampler> mDownsampler;