Merge branch 'dev' into merge-noise-branch

Author: nivram-phy

.github/workflows/async-auto-label.yml

@@ -0,0 +1,17 @@
+---
+name: Apply requested async label
+
+'on':
+  issue_comment:
+    types:
+      - created
+      - edited
+
+permissions: {}
+
+jobs:
+  apply_async_labels:
+    name: Apply requested async label
+    uses: alisw/ali-bot/.github/workflows/async-auto-label.yml@master
+    permissions:
+      pull-requests: write   # to update labels

.github/workflows/async-list-label.yml

@@ -0,0 +1,19 @@
+---
+name: Collect and print async labels
+
+'on':
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+    branches:
+      - dev
+
+permissions: {}
+
+jobs:
+  list_async_labels:
+    name: Collect and print async labels
+    uses: alisw/ali-bot/.github/workflows/async-list-label.yml@master
+    permissions:
+      pull-requests: write   # to update labels

.github/workflows/clean-test.yml

@@ -43,6 +43,15 @@ name: Clean PR checks
         description: build/O2/o2-dataflow-cs8
         type: boolean
         default: true
+      'check_build/O2/o2/aarch64':
+        description: build/O2/o2/aarch64
+        type: boolean
+        default: true
+      'check_build/O2/o2_slc9':
+        description: build/O2/o2_slc9
+        type: boolean
+        default: true
+
 
 permissions: {}
 

.github/workflows/datamodel-doc.yml

@@ -48,6 +48,7 @@ jobs:
         run: |
           python3 -m pip install --user -U numpy nltk
           python3 -m nltk.downloader -d ~/nltk_data punkt
+          python3 -m nltk.downloader -d ~/nltk_data punkt_tab
 
       - name: Generate documentation
         run: exec bash -exo pipefail O2/scripts/datamodel-doc/update-datamodel.sh
@@ -59,6 +60,7 @@ jobs:
         run: |
           # git diff --quiet exits with 1 if any tracked files have changed, and
           # with 0 otherwise.
+          set -e
           if git diff --quiet; then
             exit  # Nothing has changed, so no need to send a PR.
           fi
@@ -68,9 +70,10 @@ jobs:
           git push -f origin auto-datamodel-doc
 
           # Send pull request
-          # We need to use "hub" ourselves because alisw/pull-request gets
+          # We need to use "gh" ourselves because alisw/pull-request gets
           # confused when multiple repos are checked out.
-          hub pull-request -b AliceO2Group:master -h alibuild:auto-datamodel-doc \
-              --no-edit --no-maintainer-edits -m 'Automatic data model update'   \
-              -m "This update to the data model documentation was automatically created from tonight's O2 dev branch." ||
-            :  # If the PR already exists, hub fails, but we've just force-pushed, so we don't need a new PR.
+          GH_TOKEN="$GITHUB_TOKEN" gh pr create -R AliceO2Group/analysis-framework -B master \
+          --no-maintainer-edit -t 'Automatic data model update' -b "This update \
+          to the data model documentation was automatically created from \
+          tonight's O2 dev branch." || true
+          # If the PR already exists, hub fails, but we've just force-pushed, so we don't need a new PR.

.github/workflows/pr-security-approval.yml

@@ -0,0 +1,18 @@
+---
+name: Security approval
+
+'on':
+  pull_request_review:
+    types:
+      - edited
+      - submitted
+
+permissions: {}
+
+jobs:
+  clean:
+    name: Security approval
+    uses: alisw/ali-bot/.github/workflows/pr-security-approval.yml@master
+    permissions:
+      pull-requests: read  # to get last commit for PR
+      statuses: write      # for set-github-status

.github/workflows/reports.yml

@@ -18,10 +18,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: '3.10'
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
@@ -81,7 +81,7 @@ jobs:
       run: |
         set -x
         mkdir -p doc/data
-        # We create new files once per month, mostly so that 
+        # We create new files once per month, mostly so that
         # we can keep the query results small. It does not
         # matter if we get results from different months,
         # as what matters is how we merge them.
@@ -96,7 +96,6 @@ jobs:
         #        being published
         LAST_RELEASE="${{ github.event.inputs.LAST_RELEASE_DATE }}"
         MERGED_AFTER=${LAST_RELEASE:-$(date -v -14d +%Y-%m-%d)}
-        
         # Here we convert all the json files to per subsystem
         # logs, using the MERGED_AFTER date to further filter them.
         # Notice we can have duplicates in each file,
@@ -106,7 +105,7 @@ jobs:
         for f in doc/data/*_prs.json; do
           for x in Algorithm Analysis Common DataFormats Detectors EventVisualisation Examples Framework Generators Steer Testing Utilities; do
             cat $f | jq ".repository.pullRequests.edges[].node | select(.files.edges[].node.path | test(\"$x\")) | del(.files) | select(.state == \"MERGED\" and .mergedAt >= \"${MERGED_AFTER}\")" > /tmp/${x}_prs.json
-            if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then 
+            if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then
               cat /tmp/${x}_prs.json | jq -r '"- [#\(.number)](https://github.com/AliceO2Group/AliceO2/pull/\(.number)) \(.mergedAt | split("T")[0]): \(.title) by [@\(.author.login)](https://github.com/\(.author.login))"' | sort -u >> /tmp/${x}_prs.md
             fi
           done
@@ -127,11 +126,6 @@ jobs:
         git config --global user.name "GitHub Action Bot"
         git commit -m "Updated README" -a || echo "No changes to commit"
         git push origin HEAD:changelog -f
-        GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}  \
-          hub pull-request -f -b dev -h changelog \
-              --no-edit --no-maintainer-edits     \
-              'Auto-generated changelog'          \
-              -m 'The following changelog has been automatically generated.' ||
-            # If the PR already exists, the force-push will have updated it.
-            # It's fine if this step fails.
-            true
+        # If the PR already exists, the force-push will have updated it.
+        # It's fine if this step fails.
+        GH_TOKEN=${{ secrets.GITHUB_TOKEN }} gh pr create -B dev -H changelog -t 'Auto-generated changelog' -b 'The following changelog has been automatically generated.' || true

.gitignore

@@ -63,6 +63,7 @@ compile_commands.json
 .vscode
 .ycm_extra_conf.py
 Session.vim
+CMakeLists.txt.user
 
 # Datafiles
 gphysi.dat

CCDB/include/CCDB/BasicCCDBManager.h

@@ -18,6 +18,8 @@
 #include "CCDB/CcdbApi.h"
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "CommonUtils/NameConf.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/DefaultsHelpers.h"
 #include <string>
 #include <chrono>
 #include <map>
@@ -48,12 +50,19 @@ class CCDBManagerInstance
     std::string uuid;
     long startvalidity = 0;
     long endvalidity = -1;
+    long cacheValidFrom = 0;   // time for which the object was cached
+    long cacheValidUntil = -1; // object is guaranteed to be valid till this time (modulo new updates)
     size_t minSize = -1ULL;
     size_t maxSize = 0;
     int queries = 0;
     int fetches = 0;
     int failures = 0;
-    bool isValid(long ts) { return ts < endvalidity && ts > startvalidity; }
+    bool isValid(long ts) { return ts < endvalidity && ts >= startvalidity; }
+    bool isCacheValid(long ts)
+    {
+      LOGP(debug, "isCacheValid : {} : {} : {} --> {}", cacheValidFrom, ts, cacheValidUntil, ts < cacheValidUntil && ts >= cacheValidFrom);
+      return ts < cacheValidUntil && ts >= cacheValidFrom;
+    }
     void clear()
     {
       noCleanupPtr = nullptr;
@@ -70,6 +79,7 @@ class CCDBManagerInstance
   CCDBManagerInstance(std::string const& path) : mCCDBAccessor{}
   {
     mCCDBAccessor.init(path);
+    mDeplMode = o2::framework::DefaultsHelpers::deploymentMode();
   }
   /// set a URL to query from
   void setURL(const std::string& url);
@@ -92,6 +102,10 @@ class CCDBManagerInstance
   template <typename T>
   T* getForTimeStamp(std::string const& path, long timestamp);
 
+  /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run
+  template <typename T>
+  T* getForRun(std::string const& path, int runNumber, bool setRunMetadata = false);
+
   /// retrieve an object of type T from CCDB as stored under path, timestamp and metaData
   template <typename T>
   T* getSpecific(std::string const& path, long timestamp = -1, MD metaData = MD())
@@ -101,6 +115,13 @@ class CCDBManagerInstance
     return getForTimeStamp<T>(path, timestamp);
   }
 
+  /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run + metadata. The run number is provided separately to conform to typical analysis use (in which case metadata does not include runNumber)
+  template <typename T>
+  T* getSpecificForRun(std::string const& path, int runNumber, MD metaData = MD());
+
+  /// detect online processing modes (i.e. CCDB objects may be updated in the lifetime of the manager)
+  bool isOnline() const { return mDeplMode == o2::framework::DeploymentMode::OnlineAUX || mDeplMode == o2::framework::DeploymentMode::OnlineDDS || mDeplMode == o2::framework::DeploymentMode::OnlineECS; }
+
   /// retrieve an object of type T from CCDB as stored under path; will use the timestamp member
   template <typename T>
   T* get(std::string const& path)
@@ -134,7 +155,7 @@ class CCDBManagerInstance
     if (!isCachingEnabled()) {
       return false;
     }
-    return mCache[path].isValid(timestamp);
+    return (mCheckObjValidityEnabled && mCache[path].isValid(timestamp)) || mCache[path].isCacheValid(timestamp); // use stricter check
   }
 
   /// check if checks of object validity before CCDB query is enabled
@@ -167,15 +188,13 @@ class CCDBManagerInstance
   void setFatalWhenNull(bool b) { mFatalWhenNull = b; }
 
   /// A convenience function for MC to fetch
-  /// valid start and end timestamps given an ALICE run number.
-  /// On error it fatals (if fatal == true) or else returns the pair -1, -1.
-  std::pair<int64_t, int64_t> getRunDuration(int runnumber, bool fatal = true) const;
-
-  /// A convenience function for MC to fetch
-  /// valid start and end timestamps given an ALICE run number.
+  /// valid start and end timestamps for recorded TF data given an ALICE run number.
+  /// In absence of STF/ETF fields in the RCT with fall back to CTP SOX/EOX then to
+  /// ECS SOR/EOR.
   /// On error it fatals (if fatal == true) or else returns the pair -1, -1.
+  std::pair<int64_t, int64_t> getRunDuration(int runnumber, bool fatal = true);
   static std::pair<int64_t, int64_t> getRunDuration(o2::ccdb::CcdbApi const& api, int runnumber, bool fatal = true);
-
+  static std::pair<int64_t, int64_t> getRunDuration(const MD& headers);
   std::string getSummaryString() const;
 
   size_t getFetchedSize() const { return mFetchedSize; }
@@ -204,7 +223,7 @@ class CCDBManagerInstance
   int mQueries = 0;                                     // total number of object queries
   int mFetches = 0;                                     // total number of succesful fetches from CCDB
   int mFailures = 0;                                    // total number of failed fetches
-
+  o2::framework::DeploymentMode mDeplMode;              // O2 deployment mode
   ClassDefNV(CCDBManagerInstance, 1);
 };
 
@@ -234,7 +253,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
   } else {
     auto& cached = mCache[path];
     cached.queries++;
-    if (mCheckObjValidityEnabled && cached.isValid(timestamp)) {
+    if ((!isOnline() && cached.isCacheValid(timestamp)) || (mCheckObjValidityEnabled && cached.isValid(timestamp))) {
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
@@ -250,6 +269,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         cached.objPtr.reset(ptr);
       }
       cached.uuid = mHeaders["ETag"];
+
       try {
         if (mHeaders.find("Valid-From") != mHeaders.end()) {
           cached.startvalidity = std::stol(mHeaders["Valid-From"]);
@@ -263,6 +283,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         } else {
           cached.endvalidity = std::numeric_limits<long>::max();
         }
+        cached.cacheValidFrom = timestamp;
       } catch (std::exception const& e) {
         reportFatal("Failed to read validity from CCDB response (Valid-From :  " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")"));
       }
@@ -276,8 +297,13 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;
       cached.clear(); // in case of any error clear cache for this object
-    } else {          // the old object is valid
-      ptr = reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
+    }
+    // the old object is valid, fetch cache end of validity
+    ptr = reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
+    if (mHeaders.find("Cache-Valid-Until") != mHeaders.end()) {
+      cached.cacheValidUntil = std::stol(mHeaders["Cache-Valid-Until"]);
+    } else {
+      cached.cacheValidUntil = -1;
     }
     mHeaders.clear();
     mMetaData.clear();
@@ -293,6 +319,27 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
   return ptr;
 }
 
+template <typename T>
+T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool setRunMetadata)
+{
+  auto metaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{};
+  mMetaData = metaData;
+  return getSpecificForRun<T>(path, runNumber, metaData);
+}
+
+template <typename T>
+T* CCDBManagerInstance::getSpecificForRun(std::string const& path, int runNumber, MD metaData)
+{
+  auto [start, stop] = getRunDuration(runNumber);
+  if (start < 0 || stop < 0) {
+    if (mFatalWhenNull) {
+      reportFatal(std::string("Failed to get run duration for run ") + std::to_string(runNumber));
+    }
+    return nullptr;
+  }
+  return getSpecific<T>(path, start / 2 + stop / 2, metaData);
+}
+
 class BasicCCDBManager : public CCDBManagerInstance
 {
  public:

CCDB/include/CCDB/CCDBDownloader.h

@@ -52,6 +52,7 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
+  curl_slist* optionsList;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -86,6 +87,11 @@ curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct cur
  */
 void onUVClose(uv_handle_t* handle);
 
+enum DownloaderErrorLevel {
+  MINOR,
+  SEVERE
+};
+
 /// A class encapsulating and performing simple CURL requests in terms of a so-called CURL multi-handle.
 /// A multi-handle allows to use a connection pool (connection cache) in the CURL layer even
 /// with short-lived CURL easy-handles. Thereby the overhead of connection to servers can be
@@ -296,6 +302,7 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
+    curl_slist** options;
   } PerformData;
 #endif
 
@@ -421,6 +428,6 @@ typedef struct DataForClosingSocket {
   curl_socket_t socket;
 } DataForClosingSocket;
 
-} // namespace o2
+} // namespace o2::ccdb
 
 #endif // O2_CCDB_CCDBDOWNLOADER_H