Merge branch 'dev' into filtered-attach

Author: aalkin

.github/workflows/clean-test.yml

@@ -19,10 +19,6 @@ name: Clean PR checks
       # Warning: the check_* keys are magic and must consist of the string
       # "check_" followed by the applicable check name exactly. The
       # "description" field is only the human-readable label for the input.
-      'check_build/O2/o2':
-        description: build/O2/o2
-        type: boolean
-        default: true
       'check_build/AliceO2/O2/o2/macOS':
         description: build/AliceO2/O2/o2/macOS
         type: boolean
@@ -31,14 +27,10 @@ name: Clean PR checks
         description: build/AliceO2/O2/o2/macOS-arm
         type: boolean
         default: true
-      'check_build/O2/fullCI':
+      'check_build/O2/fullCI_slc9':
         description: build/O2/fullCI
         type: boolean
         default: true
-      'check_build/O2/o2-cs8':
-        description: build/O2/o2-cs8
-        type: boolean
-        default: true
       'check_build/O2/o2-dataflow-cs8':
         description: build/O2/o2-dataflow-cs8
         type: boolean

.github/workflows/reports.yml

@@ -17,12 +17,12 @@ jobs:
     if: github.repository == 'AliceO2Group/AliceO2'
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       name: Configure pip caching
       with:
         path: ~/.cache/pip

CCDB/include/CCDB/BasicCCDBManager.h

@@ -330,10 +330,10 @@ T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool s
 template <typename T>
 T* CCDBManagerInstance::getSpecificForRun(std::string const& path, int runNumber, MD metaData)
 {
-  auto [start, stop] = getRunDuration(runNumber);
+  auto [start, stop] = getRunDuration(runNumber, mFatalWhenNull);
   if (start < 0 || stop < 0) {
     if (mFatalWhenNull) {
-      reportFatal(std::string("Failed to get run duration for run ") + std::to_string(runNumber));
+      reportFatal(std::string("Failed to get run duration for run ") + std::to_string(runNumber) + std::string(" from CCDB"));
     }
     return nullptr;
   }

CCDB/include/CCDB/CCDBDownloader.h

@@ -47,6 +47,7 @@ struct HeaderObjectPair_t {
 
 typedef struct DownloaderRequestData {
   std::vector<std::string> hosts;
+  std::vector<std::string> locations;
   std::string path;
   long timestamp;
   HeaderObjectPair_t hoPair;
@@ -231,12 +232,13 @@ class CCDBDownloader
   std::string prepareRedirectedURL(std::string address, std::string potentialHost) const;
 
   /**
-   * Returns a vector of possible content locations based on the redirect headers.
+   * Updates the locations vector with the the locations.
    *
-   * @param baseUrl Content path.
    * @param headerMap Map containing response headers.
+   * @param locations Location list to be updated.
+   * @param locIndex Index of the next locaiton to be tried.
    */
-  std::vector<std::string> getLocations(std::multimap<std::string, std::string>* headerMap) const;
+  void updateLocations(std::multimap<std::string, std::string>* headerMap, std::vector<std::string>* locations, int* locIndex) const;
 
   std::string mUserAgentId = "CCDBDownloader";
   /**

CCDB/include/CCDB/CcdbApi.h

@@ -348,6 +348,11 @@ class CcdbApi //: public DatabaseInterface
   TObject* retrieveFromTFile(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp,
                              std::map<std::string, std::string>* headers, std::string const& etag,
                              const std::string& createdNotAfter, const std::string& createdNotBefore) const;
+  void loadFileToMemory(std::vector<char>& dest, std::string const& path,
+                        std::map<std::string, std::string> const& metadata, long timestamp,
+                        std::map<std::string, std::string>* headers, std::string const& etag,
+                        const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot = true) const;
+
 #if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
   typedef struct RequestContext {
     o2::pmr::vector<char>& dest;
@@ -383,7 +388,7 @@ class CcdbApi //: public DatabaseInterface
   static bool removeSemaphore(std::string const& name, bool remove = false);
   static void removeLeakingSemaphores(std::string const& basedir, bool remove = false);
 
-  void loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders = nullptr) const;
+  void loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders = nullptr, bool fetchLocalMetaData = true) const;
   void loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& path,
                         std::map<std::string, std::string> const& metadata, long timestamp,
                         std::map<std::string, std::string>* headers, std::string const& etag,

CCDB/src/CCDBDownloader.cxx

@@ -362,7 +362,7 @@ void CCDBDownloader::tryNewHost(PerformData* performData, CURL* easy_handle)
 {
   auto requestData = performData->requestData;
   std::string newUrl = requestData->hosts.at(performData->hostInd) + "/" + requestData->path + "/" + std::to_string(requestData->timestamp);
-  LOG(debug) << "Connecting to another host " << newUrl;
+  LOG(debug) << "Connecting to another host " << newUrl << "\n";
   requestData->hoPair.header.clear();
   curl_easy_setopt(easy_handle, CURLOPT_URL, newUrl.c_str());
   mHandlesToBeAdded.push_back(easy_handle);
@@ -374,9 +374,11 @@ void CCDBDownloader::getLocalContent(PerformData* performData, std::string& newL
   LOG(debug) << "Redirecting to local content " << newLocation << "\n";
   if (requestData->localContentCallback(newLocation)) {
     contentRetrieved = true;
+    LOG(debug) << "Local content retrieved succesfully: " << newLocation << " n";
   } else {
     // Prepare next redirect url
     newLocation = getNewLocation(performData, locations);
+    LOG(debug) << "Failed to retrieve local content: " << newLocation << "\n";
   }
 }
 
@@ -396,15 +398,15 @@ std::string CCDBDownloader::getNewLocation(PerformData* performData, std::vector
 void CCDBDownloader::httpRedirect(PerformData* performData, std::string& newLocation, CURL* easy_handle)
 {
   auto requestData = performData->requestData;
-  LOG(debug) << "Trying content location " << newLocation;
+  LOG(debug) << "Trying content location " << newLocation << "\n";
   curl_easy_setopt(easy_handle, CURLOPT_URL, newLocation.c_str());
   mHandlesToBeAdded.push_back(easy_handle);
 }
 
 void CCDBDownloader::followRedirect(PerformData* performData, CURL* easy_handle, std::vector<std::string>& locations, bool& rescheduled, bool& contentRetrieved)
 {
   std::string newLocation = getNewLocation(performData, locations);
-  if (newLocation.find("alien:/", 0) != std::string::npos || newLocation.find("file:/", 0) != std::string::npos) {
+  while (!contentRetrieved && (newLocation.find("alien:/", 0) != std::string::npos || newLocation.find("file:/", 0) != std::string::npos)) {
     getLocalContent(performData, newLocation, contentRetrieved, locations);
   }
   if (!contentRetrieved && newLocation != "") {
@@ -508,17 +510,17 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
       std::string currentHost = requestData->hosts[performData->hostInd];
       std::string loggingMessage = prepareLogMessage(currentHost, requestData->userAgent, requestData->path, requestData->timestamp, requestData->headers, httpCode);
 
-      // Get alternative locations for the same host
-      auto locations = getLocations(&(requestData->hoPair.header));
+      // Get new locations based on received headers
+      updateLocations(&(requestData->hoPair.header), &requestData->locations, &performData->locInd);
 
       // React to received http code
       if (200 <= httpCode && httpCode < 400) {
         LOG(debug) << loggingMessage;
         if (304 == httpCode) {
           LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
           contentRetrieved = true;
-        } else if (300 <= httpCode && httpCode < 400 && performData->locInd < locations.size()) {
-          followRedirect(performData, easy_handle, locations, rescheduled, contentRetrieved);
+        } else if (300 <= httpCode && httpCode < 400 && performData->locInd < requestData->locations.size()) {
+          followRedirect(performData, easy_handle, requestData->locations, rescheduled, contentRetrieved);
         } else if (200 <= httpCode && httpCode < 300) {
           contentRetrieved = true; // Can be overruled by following error check
         }
@@ -531,8 +533,16 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
         contentRetrieved = false;
       }
 
-      // Check if content was retrieved, or scheduled to be retrieved
-      if (!rescheduled && !contentRetrieved && performData->locInd == locations.size()) {
+      // Check if content was retrieved or scheduled to be retrieved
+      if (!rescheduled && !contentRetrieved) {
+        // Current location failed without providing 3xx http code, try next redirect for the same host
+        if (performData->locInd < requestData->locations.size()) {
+          followRedirect(performData, easy_handle, requestData->locations, rescheduled, contentRetrieved);
+        }
+      }
+
+      // Check again because content might have been retrieved or rescheduled via a redirect
+      if (!rescheduled && !contentRetrieved) {
         // Ran out of locations to redirect, try new host
         if (++performData->hostInd < requestData->hosts.size()) {
           tryNewHost(performData, easy_handle);
@@ -650,24 +660,37 @@ CURLcode CCDBDownloader::perform(CURL* handle)
   return batchBlockingPerform(handleVector).back();
 }
 
-std::vector<std::string> CCDBDownloader::getLocations(std::multimap<std::string, std::string>* headerMap) const
+void CCDBDownloader::updateLocations(std::multimap<std::string, std::string>* headerMap, std::vector<std::string>* locations, int* locIndex) const
 {
-  std::vector<std::string> locs;
+  std::vector<std::string> newLocations;
+
   auto iter = headerMap->find("Location");
   if (iter != headerMap->end()) {
-    locs.push_back(iter->second);
+    auto range = headerMap->equal_range("Location");
+    for (auto it = range.first; it != range.second; ++it) {
+      if (std::find(locations->begin(), locations->end(), it->second) == locations->end()) {
+        if (std::find(newLocations.begin(), newLocations.end(), it->second) == newLocations.end()) {
+          newLocations.push_back(it->second);
+        }
+      }
+    }
   }
+
   // add alternative locations (not yet included)
   auto iter2 = headerMap->find("Content-Location");
   if (iter2 != headerMap->end()) {
     auto range = headerMap->equal_range("Content-Location");
     for (auto it = range.first; it != range.second; ++it) {
-      if (std::find(locs.begin(), locs.end(), it->second) == locs.end()) {
-        locs.push_back(it->second);
+      if (std::find(locations->begin(), locations->end(), it->second) == locations->end()) {
+        if (std::find(newLocations.begin(), newLocations.end(), it->second) == newLocations.end()) {
+          newLocations.push_back(it->second);
+        }
       }
     }
   }
-  return locs;
+
+  // Insert location list at the current location index. This assures that the provided locations will be tried first.
+  locations->insert(locations->begin() + (*locIndex), newLocations.begin(), newLocations.end());
 }
 
 std::vector<CURLcode> CCDBDownloader::batchBlockingPerform(std::vector<CURL*> const& handleVector)

CCDB/src/CcdbApi.cxx

@@ -667,6 +667,23 @@ size_t header_map_callback(char* buffer, size_t size, size_t nitems, void* userd
         }
       }
     }
+
+    // Keep only the first ETag encountered
+    if (key == "ETag") {
+      auto cl = headers->find("ETag");
+      if (cl != headers->end()) {
+        insert = false;
+      }
+    }
+
+    // Keep only the first Content-Type encountered
+    if (key == "Content-Type") {
+      auto cl = headers->find("Content-Type");
+      if (cl != headers->end()) {
+        insert = false;
+      }
+    }
+
     if (insert) {
       headers->insert(std::make_pair(key, value));
     }
@@ -1461,7 +1478,7 @@ std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& p
 
   if (!mSnapshotCachePath.empty()) {
     // protect this sensitive section by a multi-process named semaphore
-    auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, path);
+    auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath + std::string("_headers"), path);
 
     std::string logfile = mSnapshotCachePath + "/log";
     std::fstream out(logfile, ios_base::out | ios_base::app);
@@ -1870,6 +1887,21 @@ void CcdbApi::saveSnapshot(RequestContext& requestContext) const
   }
 }
 
+void CcdbApi::loadFileToMemory(std::vector<char>& dest, std::string const& path,
+                               std::map<std::string, std::string> const& metadata, long timestamp,
+                               std::map<std::string, std::string>* headers, std::string const& etag,
+                               const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const
+{
+  o2::pmr::vector<char> destP;
+  destP.reserve(dest.size());
+  loadFileToMemory(destP, path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore, considerSnapshot);
+  dest.clear();
+  dest.reserve(destP.size());
+  for (const auto c : destP) {
+    dest.push_back(c);
+  }
+}
+
 void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& path,
                                std::map<std::string, std::string> const& metadata, long timestamp,
                                std::map<std::string, std::string>* headers, std::string const& etag,
@@ -1956,20 +1988,32 @@ void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestConte
 bool CcdbApi::loadLocalContentToMemory(o2::pmr::vector<char>& dest, std::string& url) const
 {
   if (url.find("alien:/", 0) != std::string::npos) {
-    loadFileToMemory(dest, url, nullptr); // headers loaded from the file in case of the snapshot reading only
-    return true;
+    std::map<std::string, std::string> localHeaders;
+    loadFileToMemory(dest, url, &localHeaders, false);
+    auto it = localHeaders.find("Error");
+    if (it != localHeaders.end() && it->second == "An error occurred during retrieval") {
+      return false;
+    } else {
+      return true;
+    }
   }
   if ((url.find("file:/", 0) != std::string::npos)) {
     std::string path = url.substr(7);
     if (std::filesystem::exists(path)) {
-      loadFileToMemory(dest, path, nullptr);
-      return true;
+      std::map<std::string, std::string> localHeaders;
+      loadFileToMemory(dest, url, &localHeaders, o2::utils::Str::endsWith(path, ".root"));
+      auto it = localHeaders.find("Error");
+      if (it != localHeaders.end() && it->second == "An error occurred during retrieval") {
+        return false;
+      } else {
+        return true;
+      }
     }
   }
   return false;
 }
 
-void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders) const
+void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders, bool fetchLocalMetaData) const
 {
   // Read file to memory as vector. For special case of the locally cached file retriev metadata stored directly in the file
   constexpr size_t MaxCopySize = 0x1L << 25;
@@ -2017,7 +2061,7 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& p
     totalread += nread;
   } while (nread == (long)MaxCopySize);
 
-  if (localHeaders) {
+  if (localHeaders && fetchLocalMetaData) {
     TMemFile memFile("name", const_cast<char*>(dest.data()), dest.size(), "READ");
     auto storedmeta = (std::map<std::string, std::string>*)extractFromTFile(memFile, TClass::GetClass("std::map<std::string, std::string>"), CCDBMETA_ENTRY);
     if (storedmeta) {

CODEOWNERS

@@ -29,12 +29,12 @@
 /DataFormats/Detectors/CPV                     @peressounko @kharlov
 /DataFormats/Detectors/CTP                     @lietava
 /DataFormats/Detectors/EMCAL                   @mfasDa @jokonig
-/DataFormats/Detectors/FIT                     @jotwinow @afurs @andreasmolander @arvindkhuntia @mslupeck
+/DataFormats/Detectors/FIT                     @jotwinow @afurs @andreasmolander @sahilupadhyaya92
 /DataFormats/Detectors/FOCAL                   @maxrauch @mfasDa @iarsene @matthiasrichter
 /DataFormats/Detectors/GlobalTracking          @shahor02
 /DataFormats/Detectors/GlobalTrackingWorkflow  @shahor02
 /DataFormats/Detectors/HMPID                   @gvolpe79
-/DataFormats/Detectors/ITSMFT                  @mcoquet642 @mconcas @shahor02
+/DataFormats/Detectors/ITSMFT                  @fprino @mcoquet642 @mconcas @shahor02
 /DataFormats/Detectors/MUON                    @AliceO2Group/muon-experts @shahor02
 /DataFormats/Detectors/PHOS                    @peressounko @kharlov
 /DataFormats/Detectors/Passive                 @sawenzel
@@ -59,13 +59,13 @@
 /Detectors/Calibration             @chiarazampolli @shahor02
 /Detectors/CPV                     @peressounko @kharlov
 /Detectors/EMCAL                   @mfasDa @jokonig
-/Detectors/FIT                     @jotwinow @afurs @andreasmolander @arvindkhuntia @mslupeck
+/Detectors/FIT                     @jotwinow @afurs @andreasmolander @sahilupadhyaya92
 /Detectors/FOCAL                   @maxrauch @mfasDa @iarsene @matthiasrichter
 /Detectors/Geometry                @sawenzel @shahor02
 /Detectors/GlobalTracking          @shahor02
 /Detectors/GlobalTrackingWorkflow  @shahor02
 /Detectors/HMPID                   @gvolpe79
-/Detectors/ITSMFT                  @mcoquet642 @mconcas @shahor02
+/Detectors/ITSMFT                  @fprino @mcoquet642 @mconcas @shahor02
 /Detectors/MUON                    @AliceO2Group/muon-experts @shahor02
 /Detectors/PHOS                    @peressounko @kharlov
 /Detectors/Passive                 @sawenzel

Common/Constants/include/CommonConstants/PhysicsConstants.h

@@ -52,6 +52,7 @@ enum Pdg {
   kDS1 = 10433,
   kDS2Star = 435,
   kDStar = 413,
+  kDStar0 = 423,
   kChiC1 = 20443,
   kJPsi = 443,
   kLambdaB0 = 5122,
@@ -101,6 +102,7 @@ constexpr double MassDSStar = 2.1122;
 constexpr double MassDS1 = 2.53511;
 constexpr double MassDS2Star = 2.5691;
 constexpr double MassDStar = 2.01026;
+constexpr double MassDStar0 = 2.00685;
 constexpr double MassChiC1 = 3.51067;
 constexpr double MassJPsi = 3.0969;
 constexpr double MassLambdaB0 = 5.6196;

Common/Constants/include/CommonConstants/make_pdg_header.py

@@ -107,6 +107,7 @@ class Pdg(Enum):
     kDS1 = 10433
     kDS2Star = 435
     kDStar = 413
+    kDStar0 = 423
     kChiC1 = 20443
     kJPsi = 443
     kLambdaB0 = 5122