DPL Analysis: percolate DataOrigin so that we can use it for multiple files reading.

ktf · ktf · commit cf8bafed689c · 2025-08-27T13:26:50.000+02:00
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -124,15 +124,22 @@ void DataInputDescriptor::addFileNameHolder(FileNameHolder* fn)
   mfilenames.emplace_back(fn);
 }
 
-bool DataInputDescriptor::setFile(int counter)
+bool DataInputDescriptor::setFile(int counter, std::string_view origin)
 {
   // no files left
   if (counter >= getNumberInputfiles()) {
     return false;
   }
 
+  // In case the origin starts with a anything but AOD, we add the origin as the suffix
+  // of the filename. In the future we might expand this for proper rewriting of the
+  // filename based on the origin and the original file information.
+  std::string filename = mfilenames[counter]->fileName;
+  if (!origin.starts_with("AOD")) {
+    filename = std::regex_replace(filename, std::regex("[.]root$"), fmt::format("_{}.root", origin));
+  }
+
   // open file
-  auto filename = mfilenames[counter]->fileName;
   auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
   if (rootFS.get()) {
     if (rootFS->GetFile()->GetName() == filename) {
@@ -213,11 +220,11 @@ bool DataInputDescriptor::setFile(int counter)
   return true;
 }
 
-uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
+uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF, std::string_view origin)
 {
 
   // open file
-  if (!setFile(counter)) {
+  if (!setFile(counter, origin)) {
     return 0ul;
   }
 
@@ -229,10 +236,10 @@ uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
   return (mfilenames[counter]->listOfTimeFrameNumbers)[numTF];
 }
 
-arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF)
+arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF, std::string_view origin)
 {
   // open file
-  if (!setFile(counter)) {
+  if (!setFile(counter, origin)) {
     return {};
   }
 
@@ -246,7 +253,7 @@ arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int n
   return {fmt::format("DF_{}", mfilenames[counter]->listOfTimeFrameNumbers[numTF]), mCurrentFilesystem};
 }
 
-DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename)
+DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename, std::string_view origin)
 {
   if (!mParentFileMap) {
     // This file has no parent map
@@ -283,7 +290,7 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   mParentFile->mdefaultFilenamesPtr = new std::vector<FileNameHolder*>;
   mParentFile->mdefaultFilenamesPtr->emplace_back(makeFileNameHolder(parentFileName->GetString().Data()));
   mParentFile->fillInputfiles();
-  mParentFile->setFile(0);
+  mParentFile->setFile(0, origin);
   return mParentFile;
 }
 
@@ -427,7 +434,8 @@ struct CalculateDelta {
     mTarget += (uv_hrtime() - start);
   }
 
-  void deactivate() {
+  void deactivate()
+  {
     active = false;
   }
 
@@ -440,7 +448,8 @@ struct CalculateDelta {
 bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, std::string treename, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)
 {
   CalculateDelta t(mIOTime);
-  auto folder = getFileFolder(counter, numTF);
+  std::string origin = dh.dataOrigin.as<std::string>();
+  auto folder = getFileFolder(counter, numTF, origin);
   if (!folder.filesystem()) {
     t.deactivate();
     return false;
@@ -473,7 +482,7 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh
   if (!format) {
     t.deactivate();
     LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.path());
-    auto parentFile = getParentFile(counter, numTF, treename);
+    auto parentFile = getParentFile(counter, numTF, treename, origin);
     if (parentFile != nullptr) {
       int parentNumTF = parentFile->findDFNumber(0, folder.path());
       if (parentNumTF == -1) {
@@ -817,8 +826,9 @@ arrow::dataset::FileSource DataInputDirector::getFileFolder(header::DataHeader d
   if (!didesc) {
     didesc = mdefaultDataInputDescriptor;
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
-  return didesc->getFileFolder(counter, numTF);
+  return didesc->getFileFolder(counter, numTF, origin);
 }
 
 int DataInputDirector::getTimeFramesInFile(header::DataHeader dh, int counter)
@@ -839,8 +849,9 @@ uint64_t DataInputDirector::getTimeFrameNumber(header::DataHeader dh, int counte
   if (!didesc) {
     didesc = mdefaultDataInputDescriptor;
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
-  return didesc->getTimeFrameNumber(counter, numTF);
+  return didesc->getTimeFrameNumber(counter, numTF, origin);
 }
 
 bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)
@@ -858,6 +869,7 @@ bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh,
     didesc = mdefaultDataInputDescriptor;
     treename = aod::datamodel::getTreeName(dh);
   }
+  std::string origin = dh.dataOrigin.as<std::string>();
 
   auto result = didesc->readTree(outputs, dh, counter, numTF, treename, totalSizeCompressed, totalSizeUncompressed);
   return result;
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -64,7 +64,7 @@ class DataInputDescriptor
 
   void addFileNameHolder(FileNameHolder* fn);
   int fillInputfiles();
-  bool setFile(int counter);
+  bool setFile(int counter, std::string_view origin);
 
   // getters
   std::string getInputfilesFilename();
@@ -74,9 +74,9 @@ class DataInputDescriptor
   int getNumberTimeFrames() { return mtotalNumberTimeFrames; }
   int findDFNumber(int file, std::string dfName);
 
-  uint64_t getTimeFrameNumber(int counter, int numTF);
-  arrow::dataset::FileSource getFileFolder(int counter, int numTF);
-  DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename);
+  uint64_t getTimeFrameNumber(int counter, int numTF, std::string_view origin);
+  arrow::dataset::FileSource getFileFolder(int counter, int numTF, std::string_view origin);
+  DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename, std::string_view origin);
   int getTimeFramesInFile(int counter);
   int getReadTimeFramesInFile(int counter);
 
diff --git a/Framework/Core/include/Framework/AnalysisSupportHelpers.h b/Framework/Core/include/Framework/AnalysisSupportHelpers.h
@@ -20,8 +20,8 @@
 
 namespace o2::framework
 {
-static constexpr std::array<header::DataOrigin, 3> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}};
-static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}};
+static constexpr std::array<header::DataOrigin, 4> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};
+static constexpr std::array<header::DataOrigin, 6> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}, header::DataOrigin{"EMB"}};
 static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
 
 class DataOutputDirector;
diff --git a/Framework/Core/src/AnalysisDataModelHelpers.cxx b/Framework/Core/src/AnalysisDataModelHelpers.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/AnalysisDataModelHelpers.h"
 #include "Framework/AnalysisDataModel.h"
+#include "Framework/AnalysisSupportHelpers.h"
 #include "Framework/StringHelpers.h"
 #include "Framework/Logger.h"
 
@@ -27,7 +28,6 @@ namespace o2::aod::datamodel
 std::string getTreeName(header::DataHeader dh)
 {
   auto description = std::string(dh.dataDescription.str);
-  auto origin = std::string(dh.dataOrigin.str);
   auto iver = (float)dh.subSpecification;
 
   // lower case of first part of description
@@ -38,11 +38,15 @@ std::string getTreeName(header::DataHeader dh)
   }
 
   // add prefix according to origin
-  if (origin == "AOD") {
-    treeName = "O2" + treeName;
+  for (auto possibleOrigin : framework::AODOrigins) {
+    if (dh.dataOrigin == possibleOrigin) {
+      treeName = "O2" + treeName;
+      break;
+    }
   }
 
   // exceptions from this
+  auto origin = std::string(dh.dataOrigin.str);
   if (origin == "AOD" && description == "MCCOLLISLABEL") {
     treeName = "O2mccollisionlabel";
   }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -25,6 +25,7 @@ using namespace o2::framework::expressions;
 
 namespace o2::aod
 {
+O2ORIGIN("EMB");
 namespace skimmedExampleTrack
 {
 DECLARE_SOA_COLUMN(Pt, pt, float);   //!
@@ -49,7 +50,7 @@ struct EtaAndClsHistogramsSimple {
     }
   }
 
-  void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&)
+  void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&, aod::StoredTracksFrom<o2::aod::Hash<"EMB"_h>> const& ortherTracks)
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
@@ -72,7 +73,7 @@ struct EtaAndClsHistogramsIUSimple {
     }
   }
 
-  void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&)
+  void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&, aod::TracksIUFrom<o2::aod::Hash<"EMB"_h>> const& otherTracks)
   {
     LOGP(info, "Invoking the simple one IU");
     for (auto& track : tracks) {

Original file line number	Diff line number	Diff line change
`@@ -124,15 +124,22 @@ void DataInputDescriptor::addFileNameHolder(FileNameHolder* fn)`
`124`	`124`	`mfilenames.emplace_back(fn);`
`125`	`125`	`}`
`126`	`126`
`127`		`-bool DataInputDescriptor::setFile(int counter)`
	`127`	`+bool DataInputDescriptor::setFile(int counter, std::string_view origin)`
`128`	`128`	`{`
`129`	`129`	`// no files left`
`130`	`130`	`if (counter >= getNumberInputfiles()) {`
`131`	`131`	`return false;`
`132`	`132`	`}`
`133`	`133`
	`134`	`+ // In case the origin starts with a anything but AOD, we add the origin as the suffix`
	`135`	`+ // of the filename. In the future we might expand this for proper rewriting of the`
	`136`	`+ // filename based on the origin and the original file information.`
	`137`	`+ std::string filename = mfilenames[counter]->fileName;`
	`138`	`+ if (!origin.starts_with("AOD")) {`
	`139`	`+ filename = std::regex_replace(filename, std::regex("[.]root$"), fmt::format("_{}.root", origin));`
	`140`	`+ }`
	`141`	`+`
`134`	`142`	`// open file`
`135`		`- auto filename = mfilenames[counter]->fileName;`
`136`	`143`	`auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);`
`137`	`144`	`if (rootFS.get()) {`
`138`	`145`	`if (rootFS->GetFile()->GetName() == filename) {`
`@@ -213,11 +220,11 @@ bool DataInputDescriptor::setFile(int counter)`
`213`	`220`	`return true;`
`214`	`221`	`}`
`215`	`222`
`216`		`-uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)`
	`223`	`+uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF, std::string_view origin)`
`217`	`224`	`{`
`218`	`225`
`219`	`226`	`// open file`
`220`		`- if (!setFile(counter)) {`
	`227`	`+ if (!setFile(counter, origin)) {`
`221`	`228`	`return 0ul;`
`222`	`229`	`}`
`223`	`230`
`@@ -229,10 +236,10 @@ uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)`
`229`	`236`	`return (mfilenames[counter]->listOfTimeFrameNumbers)[numTF];`
`230`	`237`	`}`
`231`	`238`
`232`		`-arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF)`
	`239`	`+arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF, std::string_view origin)`
`233`	`240`	`{`
`234`	`241`	`// open file`
`235`		`- if (!setFile(counter)) {`
	`242`	`+ if (!setFile(counter, origin)) {`
`236`	`243`	`return {};`
`237`	`244`	`}`
`238`	`245`
`@@ -246,7 +253,7 @@ arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int n`
`246`	`253`	`return {fmt::format("DF_{}", mfilenames[counter]->listOfTimeFrameNumbers[numTF]), mCurrentFilesystem};`
`247`	`254`	`}`
`248`	`255`
`249`		`-DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename)`
	`256`	`+DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename, std::string_view origin)`
`250`	`257`	`{`
`251`	`258`	`if (!mParentFileMap) {`
`252`	`259`	`// This file has no parent map`
`@@ -283,7 +290,7 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,`
`283`	`290`	`mParentFile->mdefaultFilenamesPtr = new std::vector<FileNameHolder*>;`
`284`	`291`	`mParentFile->mdefaultFilenamesPtr->emplace_back(makeFileNameHolder(parentFileName->GetString().Data()));`
`285`	`292`	`mParentFile->fillInputfiles();`
`286`		`- mParentFile->setFile(0);`
	`293`	`+ mParentFile->setFile(0, origin);`
`287`	`294`	`return mParentFile;`
`288`	`295`	`}`
`289`	`296`
`@@ -427,7 +434,8 @@ struct CalculateDelta {`
`427`	`434`	`mTarget += (uv_hrtime() - start);`
`428`	`435`	`}`
`429`	`436`
`430`		`- void deactivate() {`
	`437`	`+ void deactivate()`
	`438`	`+ {`
`431`	`439`	`active = false;`
`432`	`440`	`}`
`433`	`441`
`@@ -440,7 +448,8 @@ struct CalculateDelta {`
`440`	`448`	`bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, std::string treename, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)`
`441`	`449`	`{`
`442`	`450`	`CalculateDelta t(mIOTime);`
`443`		`- auto folder = getFileFolder(counter, numTF);`
	`451`	`+ std::string origin = dh.dataOrigin.as<std::string>();`
	`452`	`+ auto folder = getFileFolder(counter, numTF, origin);`
`444`	`453`	`if (!folder.filesystem()) {`
`445`	`454`	`t.deactivate();`
`446`	`455`	`return false;`
`@@ -473,7 +482,7 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh`
`473`	`482`	`if (!format) {`
`474`	`483`	`t.deactivate();`
`475`	`484`	`LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.path());`
`476`		`- auto parentFile = getParentFile(counter, numTF, treename);`
	`485`	`+ auto parentFile = getParentFile(counter, numTF, treename, origin);`
`477`	`486`	`if (parentFile != nullptr) {`
`478`	`487`	`int parentNumTF = parentFile->findDFNumber(0, folder.path());`
`479`	`488`	`if (parentNumTF == -1) {`
`@@ -817,8 +826,9 @@ arrow::dataset::FileSource DataInputDirector::getFileFolder(header::DataHeader d`
`817`	`826`	`if (!didesc) {`
`818`	`827`	`didesc = mdefaultDataInputDescriptor;`
`819`	`828`	`}`
	`829`	`+ std::string origin = dh.dataOrigin.as<std::string>();`
`820`	`830`
`821`		`- return didesc->getFileFolder(counter, numTF);`
	`831`	`+ return didesc->getFileFolder(counter, numTF, origin);`
`822`	`832`	`}`
`823`	`833`
`824`	`834`	`int DataInputDirector::getTimeFramesInFile(header::DataHeader dh, int counter)`
`@@ -839,8 +849,9 @@ uint64_t DataInputDirector::getTimeFrameNumber(header::DataHeader dh, int counte`
`839`	`849`	`if (!didesc) {`
`840`	`850`	`didesc = mdefaultDataInputDescriptor;`
`841`	`851`	`}`
	`852`	`+ std::string origin = dh.dataOrigin.as<std::string>();`
`842`	`853`
`843`		`- return didesc->getTimeFrameNumber(counter, numTF);`
	`854`	`+ return didesc->getTimeFrameNumber(counter, numTF, origin);`
`844`	`855`	`}`
`845`	`856`
`846`	`857`	`bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed)`
`@@ -858,6 +869,7 @@ bool DataInputDirector::readTree(DataAllocator& outputs, header::DataHeader dh,`
`858`	`869`	`didesc = mdefaultDataInputDescriptor;`
`859`	`870`	`treename = aod::datamodel::getTreeName(dh);`
`860`	`871`	`}`
	`872`	`+ std::string origin = dh.dataOrigin.as<std::string>();`
`861`	`873`
`862`	`874`	`auto result = didesc->readTree(outputs, dh, counter, numTF, treename, totalSizeCompressed, totalSizeUncompressed);`
`863`	`875`	`return result;`
Original file line number	Diff line number	Diff line change
`@@ -20,8 +20,8 @@`
`20`	`20`
`21`	`21`	`namespace o2::framework`
`22`	`22`	`{`
`23`		`-static constexpr std::array<header::DataOrigin, 3> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}};`
`24`		`-static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}};`
	`23`	`+static constexpr std::array<header::DataOrigin, 4> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};`
	`24`	`+static constexpr std::array<header::DataOrigin, 6> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}, header::DataOrigin{"EMB"}};`
`25`	`25`	`static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};`
`26`	`26`
`27`	`27`	`class DataOutputDirector;`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ using namespace o2::framework::expressions;`
`25`	`25`
`26`	`26`	`namespace o2::aod`
`27`	`27`	`{`
	`28`	`+O2ORIGIN("EMB");`
`28`	`29`	`namespace skimmedExampleTrack`
`29`	`30`	`{`
`30`	`31`	`DECLARE_SOA_COLUMN(Pt, pt, float); //!`
`@@ -49,7 +50,7 @@ struct EtaAndClsHistogramsSimple {`
`49`	`50`	`}`
`50`	`51`	`}`
`51`	`52`
`52`		`- void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&)`
	`53`	`+ void process(soa::Filtered<aod::Tracks> const& tracks, aod::FT0s const&, aod::StoredTracksFrom<o2::aod::Hash<"EMB"_h>> const& ortherTracks)`
`53`	`54`	`{`
`54`	`55`	`LOGP(info, "Invoking the simple one");`
`55`	`56`	`for (auto& track : tracks) {`
`@@ -72,7 +73,7 @@ struct EtaAndClsHistogramsIUSimple {`
`72`	`73`	`}`
`73`	`74`	`}`
`74`	`75`
`75`		`- void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&)`
	`76`	`+ void process(soa::Filtered<aod::TracksIU> const& tracks, aod::FT0s const&, aod::TracksIUFrom<o2::aod::Hash<"EMB"_h>> const& otherTracks)`
`76`	`77`	`{`
`77`	`78`	`LOGP(info, "Invoking the simple one IU");`
`78`	`79`	`for (auto& track : tracks) {`