Extend CCDB functions with optional headers

Author: mrtineide

CCDB/CMakeLists.txt

@@ -92,6 +92,12 @@ o2_add_test(CcdbDownloader
             PUBLIC_LINK_LIBRARIES O2::CCDB
             LABELS ccdb)
 
+o2_add_test(CcdbApi-Headers
+            SOURCES test/testCcdbApiHeaders.cxx
+            COMPONENT_NAME ccdb
+            PUBLIC_LINK_LIBRARIES O2::CCDB
+            LABELS ccdb)
+
 # extra CcdbApi test which dispatches to CCDBDownloader (tmp until full move done)
 #o2_add_test_command(NAME CcdbApi-MultiHandle
 #                    WORKING_DIRECTORY ${SIMTESTDIR}

CCDB/include/CCDB/BasicCCDBManager.h

@@ -23,9 +23,11 @@
 #include <string>
 #include <chrono>
 #include <map>
+#include <string_view>
 #include <unordered_map>
 #include <memory>
 #include <cstdlib>
+#include <vector>
 
 class TGeoManager; // we need to forward-declare those classes which should not be cleaned up
 
@@ -57,6 +59,7 @@ class CCDBManagerInstance
     int queries = 0;
     int fetches = 0;
     int failures = 0;
+    std::map<std::string, std::string> cacheOfHeaders;
     bool isValid(long ts) { return ts < endvalidity && ts >= startvalidity; }
     bool isCacheValid(long ts)
     {
@@ -70,6 +73,7 @@ class CCDBManagerInstance
       uuid = "";
       startvalidity = 0;
       endvalidity = -1;
+      cacheOfHeaders.clear();
     }
   };
 
@@ -98,9 +102,9 @@ class CCDBManagerInstance
   /// query timestamp
   long getTimestamp() const { return mTimestamp; }
 
-  /// retrieve an object of type T from CCDB as stored under path and timestamp
+  /// retrieve an object of type T from CCDB as stored under path and timestamp. Optional to get the headers. Can give a filter of headers to be saved in cache and returned (if present)
   template <typename T>
-  T* getForTimeStamp(std::string const& path, long timestamp);
+  T* getForTimeStamp(std::string const& path, long timestamp, std::map<std::string, std::string>* headers = nullptr, std::vector<std::string_view> headerFilter = {});
 
   /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run
   template <typename T>
@@ -112,10 +116,7 @@ class CCDBManagerInstance
   {
     // TODO: add some error info/handling when failing
     mMetaData = metaData;
-    auto obj = getForTimeStamp<T>(path, timestamp);
-    if (headers) {
-      *headers = mHeaders;
-    }
+    auto obj = getForTimeStamp<T>(path, timestamp, headers);
     return obj;
   }
 
@@ -235,7 +236,7 @@ class CCDBManagerInstance
 };
 
 template <typename T>
-T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
+T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, std::map<std::string, std::string>* headers, std::vector<std::string_view> headerFilter)
 {
   mHeaders.clear(); // we clear at the beginning; to allow to retrieve the header information in a subsequent call
   T* ptr = nullptr;
@@ -258,15 +259,45 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         mFetchedSize += s;
       }
     }
+    if (!headerFilter.empty()) {
+      LOGP(warn, "Header filter ignored when caching is disabled, giving back all headers");
+    }
+    if (headers) {
+      *headers = mHeaders;
+    }
   } else {
     auto& cached = mCache[path];
     cached.queries++;
     if ((!isOnline() && cached.isCacheValid(timestamp)) || (mCheckObjValidityEnabled && cached.isValid(timestamp))) {
+      // Give back the cached/saved headers
+      if (headers) {
+        *headers = cached.cacheOfHeaders;
+      }
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
                                                 mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "",
                                                 mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : "");
+
+    // Cache the headers
+    if (headerFilter.empty()) {
+      // No filter, cache all headers
+      for (auto const& h : mHeaders) {
+        cached.cacheOfHeaders[h.first] = h.second;
+      }
+    } else {
+      // Cache only the asked for headers
+      for (auto const& k : headerFilter) {
+        auto it = mHeaders.find(std::string(k));
+        if (it != mHeaders.end()) {
+          cached.cacheOfHeaders.insert_or_assign(it->first, it->second); // Only want to overwrite if the header exists in the source
+        }
+      }
+    }
+    if (headers) {
+      *headers = cached.cacheOfHeaders;
+    }
+
     if (ptr) { // new object was shipped, old one (if any) is not valid anymore
       cached.fetches++;
       mFetches++;
@@ -300,7 +331,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         size_t s = atol(sh->second.c_str());
         mFetchedSize += s;
         cached.minSize = std::min(s, cached.minSize);
-        cached.maxSize = std::max(s, cached.minSize);
+        cached.maxSize = std::max(s, cached.minSize); // I think this should be maxSize, not minSize
       }
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;

CCDB/include/CCDB/CcdbApi.h

@@ -281,7 +281,7 @@ class CcdbApi //: public DatabaseInterface
    *  @return: True in case operation successful or false if there was a failure/problem.
    */
   bool retrieveBlob(std::string const& path, std::string const& targetdir, std::map<std::string, std::string> const& metadata, long timestamp,
-                    bool preservePathStructure = true, std::string const& localFileName = "snapshot.root", std::string const& createdNotAfter = "", std::string const& createdNotBefore = "") const;
+                    bool preservePathStructure = true, std::string const& localFileName = "snapshot.root", std::string const& createdNotAfter = "", std::string const& createdNotBefore = "", std::map<std::string, std::string>* headers = nullptr) const;
 
   /**
    * Retrieve the headers of a CCDB entry, if it exists.

CCDB/src/CcdbApi.cxx

@@ -831,7 +831,7 @@ TObject* CcdbApi::retrieveFromTFile(std::string const& path, std::map<std::strin
 }
 
 bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir, std::map<std::string, std::string> const& metadata,
-                           long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore) const
+                           long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore, std::map<std::string, std::string>* outHeaders) const
 {
 
   // we setup the target path for this blob
@@ -879,6 +879,9 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir
   CCDBQuery querysummary(path, metadata, timestamp);
 
   updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary);
+  if (outHeaders) {
+    *outHeaders = std::move(headers); // Re-use the same headers to give back to the callee
+  }
   return true;
 }