DPL Analysis: percolate DataOrigin so that we can use it for multiple files reading.

Author: ktf

Framework/AnalysisSupport/src/DataInputDirector.cxx

@@ -124,15 +124,23 @@ void DataInputDescriptor::addFileNameHolder(FileNameHolder* fn)
   mfilenames.emplace_back(fn);
 }
 
-bool DataInputDescriptor::setFile(int counter)
+bool DataInputDescriptor::setFile(int counter, std::string_view origin)
 {
   // no files left
   if (counter >= getNumberInputfiles()) {
     return false;
   }
 
+  // In case the origin starts with a anything but AOD, we add the origin as the suffix
+  // of the filename. In the future we might expand this for proper rewriting of the
+  // filename based on the origin and the original file information.
+  std::string filename = mfilenames[counter]->fileName;
+  if (!origin.starts_with("AOD")) {
+    filename = std::regex_replace(filename, std::regex("[.]root$"), fmt::format("_{}.root", origin));
+    std::cout << "Rewriting filename to " << filename << std::endl;
+  }
+
   // open file
-  auto filename = mfilenames[counter]->fileName;
   auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
   if (rootFS.get()) {
     if (rootFS->GetFile()->GetName() == filename) {
@@ -213,11 +221,11 @@ bool DataInputDescriptor::setFile(int counter)
   return true;
 }
 
-uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
+uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF, std::string_view origin)
 {
 
   // open file
-  if (!setFile(counter)) {
+  if (!setFile(counter, origin)) {
     return 0ul;
   }
 
@@ -229,10 +237,10 @@ uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
   return (mfilenames[counter]->listOfTimeFrameNumbers)[numTF];
 }
 
-arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF)
+arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF, std::string_view origin)
 {
   // open file
-  if (!setFile(counter)) {
+  if (!setFile(counter, origin)) {
     return {};
   }
 
@@ -246,7 +254,7 @@ arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int n
   return {fmt::format("DF_{}", mfilenames[counter]->listOfTimeFrameNumbers[numTF]), mCurrentFilesystem};
 }
 
-DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename)
+DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename, std::string_view origin)
 {
   if (!mParentFileMap) {
     // This file has no parent map
@@ -283,7 +291,7 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   mParentFile->mdefaultFilenamesPtr = new std::vector<FileNameHolder*>;
   mParentFile->mdefaultFilenamesPtr->emplace_back(makeFileNameHolder(parentFileName->GetString().Data()));
   mParentFile->fillInputfiles();
-  mParentFile->setFile(0);
+  mParentFile->setFile(0, origin);
   return mParentFile;
 }
 
@@ -440,7 +448,8 @@ struct CalculateDelta {
 bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, std::string treename, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)
 {
   CalculateDelta t(mIOTime);
-  auto folder = getFileFolder(counter, numTF);
+  std::string origin = dh.dataOrigin.as<std::string>();
+  auto folder = getFileFolder(counter, numTF, origin);
   if (!folder.filesystem()) {
     t.deactivate();
     return false;
@@ -473,7 +482,7 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh
   if (!format) {
     t.deactivate();
     LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.path());
-    auto parentFile = getParentFile(counter, numTF, treename);
+    auto parentFile = getParentFile(counter, numTF, treename, origin);
     if (parentFile != nullptr) {
       int parentNumTF = parentFile->findDFNumber(0, folder.path());
       if (parentNumTF == -1) {
@@ -817,8 +826,9 @@ arrow::dataset::FileSource DataInputDirector::getFileFolder(header::DataHeader d
   if (!didesc) {
     didesc = mdefaultDataInputDescriptor;
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
-  return didesc->getFileFolder(counter, numTF);
+  return didesc->getFileFolder(counter, numTF, origin);
 }
 
 int DataInputDirector::getTimeFramesInFile(header::DataHeader dh, int counter)
@@ -839,8 +849,9 @@ uint64_t DataInputDirector::getTimeFrameNumber(header::DataHeader dh, int counte
   if (!didesc) {
     didesc = mdefaultDataInputDescriptor;
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
-  return didesc->getTimeFrameNumber(counter, numTF);
+  return didesc->getTimeFrameNumber(counter, numTF, origin);
 }
 
 bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)
@@ -858,6 +869,7 @@ bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh,
     didesc = mdefaultDataInputDescriptor;
     treename = aod::datamodel::getTreeName(dh);
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
   auto result = didesc->readTree(outputs, dh, counter, numTF, treename, totalSizeCompressed, totalSizeUncompressed);
   return result;

Framework/AnalysisSupport/src/DataInputDirector.h

@@ -64,7 +64,7 @@ class DataInputDescriptor
 
   void addFileNameHolder(FileNameHolder* fn);
   int fillInputfiles();
-  bool setFile(int counter);
+  bool setFile(int counter, std::string_view origin);
 
   // getters
   std::string getInputfilesFilename();
@@ -74,9 +74,9 @@ class DataInputDescriptor
   int getNumberTimeFrames() { return mtotalNumberTimeFrames; }
   int findDFNumber(int file, std::string dfName);
 
-  uint64_t getTimeFrameNumber(int counter, int numTF);
-  arrow::dataset::FileSource getFileFolder(int counter, int numTF);
-  DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename);
+  uint64_t getTimeFrameNumber(int counter, int numTF, std::string_view origin);
+  arrow::dataset::FileSource getFileFolder(int counter, int numTF, std::string_view origin);
+  DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename, std::string_view origin);
   int getTimeFramesInFile(int counter);
   int getReadTimeFramesInFile(int counter);
 

Framework/Core/include/Framework/AnalysisSupportHelpers.h

@@ -20,8 +20,8 @@
 
 namespace o2::framework
 {
-static constexpr std::array<header::DataOrigin, 3> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}};
-static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}};
+static constexpr std::array<header::DataOrigin, 4> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};
+static constexpr std::array<header::DataOrigin, 6> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}, header::DataOrigin{"EMB"}};
 static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
 
 class DataOutputDirector;

Framework/TestWorkflows/src/o2TestHistograms.cxx

@@ -25,6 +25,7 @@ using namespace o2::framework::expressions;
 
 namespace o2::aod
 {
+O2ORIGIN("EMB");
 namespace skimmedExampleTrack
 {
 DECLARE_SOA_COLUMN(Pt, pt, float);   //!
@@ -49,7 +50,7 @@ struct EtaAndClsHistogramsSimple {
     }
   }
 
-  void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&)
+  void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&, aod::StoredTracksFrom<o2::aod::Hash<"EMB"_h>> const& ortherTracks)
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
@@ -72,7 +73,7 @@ struct EtaAndClsHistogramsIUSimple {
     }
   }
 
-  void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&)
+  void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&, aod::TracksIUFrom<o2::aod::Hash<"EMB"_h>> const &otherTracks)
   {
     LOGP(info, "Invoking the simple one IU");
     for (auto& track : tracks) {