DPL Analysis: remove special origins and From<> version for auto-generated tables (extended, index and timestamped)

Author: aalkin

Framework/AnalysisSupport/src/AODWriterHelpers.cxx

@@ -21,7 +21,6 @@
 #include "Framework/TableConsumer.h"
 #include "Framework/DataOutputDirector.h"
 #include "Framework/TableTreeHelpers.h"
-#include "Framework/Monitoring.h"
 #include "Framework/Signpost.h"
 
 #include <Monitoring/Monitoring.h>
@@ -32,8 +31,6 @@
 #include <TMap.h>
 #include <TObjString.h>
 #include <arrow/table.h>
-#include <chrono>
-#include <ios>
 
 O2_DECLARE_DYNAMIC_LOG(histogram_registry);
 
@@ -157,7 +154,7 @@ AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx)
         }
 
         // skip non-AOD refs
-        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
+        if (!DataSpecUtils::partialMatch(*ref.spec, AODOrigins)) {
           continue;
         }
         startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;

Framework/Core/include/Framework/ASoA.h

@@ -51,6 +51,8 @@ DECLARE_SOA_TABLE(BCFlags, "AOD", "BCFLAG", //! flag for tagging UPCs, joinable
                   bc::Flags);
 
 using BCs = BCs_001; // current version
+template <aod::is_origin_hash O>
+using BCsFrom = BCs_001From<O>;
 using BC = BCs::iterator;
 
 namespace timestamp
@@ -66,7 +68,7 @@ using BCsWithTimestamps = soa::Join<aod::BCs, aod::Timestamps>;
 
 namespace soa
 {
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::BCs, aod::Timestamps>;
+extern template struct Join<aod::BCs, aod::Timestamps>;
 }
 namespace aod
 {
@@ -514,7 +516,7 @@ DECLARE_SOA_TABLE_FULL(StoredTracksIU, "Tracks_IU", "AOD", "TRACK_IU", //! On di
                        track::Sign<track::Signed1Pt>,
                        o2::soa::Marker<2>);
 
-DECLARE_SOA_EXTENDED_TABLE(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
+DECLARE_SOA_EXTENDED_TABLE_NG(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
                            aod::track::Pt,
                            aod::track::P,
                            aod::track::Eta,
@@ -680,9 +682,9 @@ using Run2TrackExtra = Run2TrackExtras::iterator;
 } // namespace aod
 namespace soa
 {
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksExtra>;
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksCov, aod::TracksExtra>;
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::TracksExtension, aod::StoredTracks>;
+extern template struct soa::Join<aod::Tracks, aod::TracksExtra>;
+extern template struct soa::Join<aod::Tracks, aod::TracksCov, aod::TracksExtra>;
+extern template struct soa::Join<aod::TracksExtension, aod::StoredTracks>;
 } // namespace soa
 namespace aod
 {
@@ -926,6 +928,8 @@ using MFTTracks = MFTTracks_001;
 using StoredMFTTracks = StoredMFTTracks_001;
 
 using MFTTrack = MFTTracks::iterator;
+template <aod::is_origin_hash O>
+using MFTTracksFrom = MFTTracks_001From<O>;
 
 namespace fwdtrack // Index to MFTtrack column must be defined after table definition.
 {
@@ -1005,7 +1009,7 @@ using MFTTrackCovFwd = MFTTracksCov::iterator;
 } // namespace aod
 namespace soa
 {
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::FwdTracks, aod::FwdTracksCov>;
+extern template struct Join<aod::FwdTracks, aod::FwdTracksCov>;
 }
 namespace aod
 {
@@ -2026,6 +2030,8 @@ DECLARE_SOA_EXTENDED_TABLE(McParticles_001, StoredMcParticles_001, "EXMCPARTICLE
 using StoredMcParticles = StoredMcParticles_001;
 using McParticles = McParticles_001;
 using McParticle = McParticles::iterator;
+template <aod::is_origin_hash O>
+using McParticlesFrom = McParticles_001From<O>;
 } // namespace aod
 namespace soa
 {
@@ -2191,11 +2197,11 @@ DECLARE_SOA_INDEX_COLUMN(FDD, fdd);                    //!
 // First entry: Collision
 #define INDEX_LIST_RUN2 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FV0CId, indices::FDDId
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run2MatchedExclusive, BCs, "MA_RN2_EX", INDEX_LIST_RUN2); //!
-DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs, "MA_RN2_SP", INDEX_LIST_RUN2);              //!
+DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs_001, "MA_RN2_SP", INDEX_LIST_RUN2);          //!
 
 #define INDEX_LIST_RUN3 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FDDId
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run3MatchedExclusive, BCs, "MA_RN3_EX", INDEX_LIST_RUN3); //!
-DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs, "MA_RN3_SP", INDEX_LIST_RUN3);              //!
+DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs_001, "MA_RN3_SP", INDEX_LIST_RUN3);          //!
 
 // First entry: BC
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(MatchedBCCollisionsExclusive, BCs, "MA_BCCOL_EX", //!
@@ -2225,8 +2231,8 @@ DECLARE_EQUIVALENT_FOR_INDEX(aod::StoredTracksIU, aod::McTrackLabels);
 DECLARE_EQUIVALENT_FOR_INDEX(aod::Collisions, aod::McCollisionLabels);
 // Joins with collisions (only for sparse ones)
 // NOTE: index table needs to be always last argument
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run2MatchedSparse>;
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run3MatchedSparse>;
+extern template struct Join<aod::Collisions, aod::Run2MatchedSparse>;
+extern template struct Join<aod::Collisions, aod::Run3MatchedSparse>;
 } // namespace soa
 namespace aod
 {

Framework/Core/include/Framework/AnalysisDataModel.h

@@ -79,6 +79,7 @@ auto makeEmptyTable(const char* name)
 }
 
 template <soa::TableRef R>
+  requires(soa::not_void<typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata>)
 auto makeEmptyTable()
 {
   auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata::persistent_columns_t{}));
@@ -93,6 +94,7 @@ auto makeEmptyTable(const char* name, framework::pack<Cs...> p)
 }
 
 template <aod::is_aod_hash D>
+  requires(soa::not_void<typename aod::MetadataTrait<D>::metadata>)
 auto makeEmptyTable(const char* name)
 {
   auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<D>::metadata::persistent_columns_t{}));
@@ -216,6 +218,26 @@ inline constexpr auto getSourceSchemas()
   }.template operator()<T::sources.size(), T::sources>();
 }
 
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+inline constexpr auto getSources()
+{
+  return []<size_t N, std::array<soa::TableRef, N> refs>() {
+    return []<size_t... Is>(std::index_sequence<Is...>) {
+      return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
+    }(std::make_index_sequence<N>());
+  }.template operator()<T::N, T::template generateSources<O>()>();
+}
+
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+inline constexpr auto getSourceSchemas()
+{
+  return []<size_t N, std::array<soa::TableRef, N> refs>() {
+    return []<size_t... Is>(std::index_sequence<Is...>) {
+      return std::vector{soa::tableRef2Schema<refs[Is]>()...};
+    }(std::make_index_sequence<N>());
+  }.template operator()<T::N, T::template generateSources<O>()>();
+}
+
 template <soa::with_ccdb_urls T>
 inline constexpr auto getCCDBUrls()
 {
@@ -257,7 +279,7 @@ inline constexpr auto getIndexMapping()
   using indices = T::index_pack_t;
   using Key = T::Key;
   [&idx]<size_t... Is>(std::index_sequence<Is...>) mutable {
-    constexpr auto refs = T::sources;
+    constexpr auto refs = T::generateSources();
     ([&idx]<TableRef ref, typename C>() mutable {
       constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<ref.desc_hash>>::metadata::template getIndexPosToKey<Key>();
       if constexpr (pos == -1) {
@@ -272,6 +294,26 @@ inline constexpr auto getIndexMapping()
   return idx;
 }
 
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
+{
+  std::vector<framework::ConfigParamSpec> inputMetadata;
+
+  auto inputSources = getSources<T, O>();
+  std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
+  auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
+  inputSources.erase(last, inputSources.end());
+  inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
+
+  auto inputSchemas = getSourceSchemas<T, O>();
+  std::sort(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
+  last = std::unique(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
+  inputSchemas.erase(last, inputSchemas.end());
+  inputMetadata.insert(inputMetadata.end(), inputSchemas.begin(), inputSchemas.end());
+
+  return inputMetadata;
+}
+
 template <soa::with_sources T>
 constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 {
@@ -293,7 +335,7 @@ constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 }
 
 template <typename T>
-  requires(!soa::with_sources<T>)
+  requires(!(soa::with_sources<T> || soa::with_sources_generator<T>))
 constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 {
   return {};
@@ -358,7 +400,12 @@ template <TableRef R>
 constexpr auto tableRef2InputSpec()
 {
   std::vector<framework::ConfigParamSpec> metadata;
-  auto sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  std::vector<framework::ConfigParamSpec> sources;
+  if constexpr (soa::with_sources<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
+    sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  } else if constexpr(soa::with_sources_generator<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
+    sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata, o2::aod::Hash<R.origin_hash>>();
+  }
   metadata.insert(metadata.end(), sources.begin(), sources.end());
   auto ccdbURLs = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
   metadata.insert(metadata.end(), ccdbURLs.begin(), ccdbURLs.end());
@@ -555,7 +602,7 @@ concept is_produces_group = std::derived_from<T, ProducesGroup>;
 template <soa::is_metadata M, soa::TableRef Ref>
 struct TableTransform {
   using metadata = M;
-  constexpr static auto sources = M::sources;
+  constexpr static auto sources = M::template generateSources<o2::aod::Hash<Ref.origin_hash>>();
 
   template <soa::TableRef R>
   static auto base_spec()
@@ -589,23 +636,23 @@ struct TableTransform {
 /// This helper struct allows you to declare extended tables which should be
 /// created by the task (as opposed to those pre-defined by data model)
 template <typename T>
-concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
+concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;
 
 template <typename T>
-concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
+concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;
 
 template <is_spawnable T>
 constexpr auto transformBase()
 {
-  using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
-  return TableTransform<metadata, metadata::extension_table_t::ref>{};
+  using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata;
+  return TableTransform<metadata, metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>::ref>{};
 }
 
 template <is_spawnable T>
 struct Spawns : decltype(transformBase<T>()) {
   using spawnable_t = T;
   using metadata = decltype(transformBase<T>())::metadata;
-  using extension_t = typename metadata::extension_table_t;
+  using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
   using expression_pack_t = typename metadata::expression_pack_t;
   static constexpr size_t N = framework::pack_size(expression_pack_t{});
 
@@ -655,7 +702,7 @@ struct Defines : decltype(transformBase<T>()) {
   static constexpr bool delayed = DELAYED;
   using spawnable_t = T;
   using metadata = decltype(transformBase<T>())::metadata;
-  using extension_t = typename metadata::extension_table_t;
+  using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
   using placeholders_pack_t = typename metadata::placeholders_pack_t;
   static constexpr size_t N = framework::pack_size(placeholders_pack_t{});
 

Framework/Core/include/Framework/AnalysisHelpers.h

@@ -275,10 +275,10 @@ bool prepareOutput(ProcessingContext& context, T& producesGroup)
 template <is_spawns T>
 bool prepareOutput(ProcessingContext& context, T& spawns)
 {
-  using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
-  auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context), std::span{metadata::base_table_t::originalLabels});
+  using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::originals[T::spawnable_t::originals.size() - 1].desc_hash>>::metadata;
+  auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::N, metadata::template generateSources<o2::aod::Hash<T::spawnable_t::originals[T::spawnable_t::originals.size() - 1].origin_hash>>()>(context), std::span{metadata::base_table_t::originalLabels});
   if (originalTable->num_rows() == 0) {
-    originalTable = makeEmptyTable<metadata::base_table_t::ref>();
+    originalTable = makeEmptyTable("EMPTY", typename metadata::base_table_t::persistent_columns_t{});
   }
   using D = o2::aod::Hash<metadata::extension_table_t::ref.desc_hash>;
 
@@ -295,17 +295,17 @@ template <is_builds T>
 bool prepareOutput(ProcessingContext& context, T& builds)
 {
   using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::buildable_t::ref.desc_hash>>::metadata;
-  return builds.build(extractOriginals<metadata::sources.size(), metadata::sources>(context));
+  return builds.build(extractOriginals<metadata::N, metadata::template generateSources<o2::aod::Hash<T::buildable_t::ref.origin_hash>>()>(context));
 }
 
 template <is_defines T>
 bool prepareOutput(ProcessingContext& context, T& defines)
   requires(T::delayed == false)
 {
-  using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
-  auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context), std::span{metadata::base_table_t::originalLabels});
+  using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::originals[T::spawnable_t::originals.size() - 1].desc_hash>>::metadata;
+  auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::N, metadata::template generateSources<o2::aod::Hash<T::spawnable_t::originals[T::spawnable_t::originals.size() - 1].origin_hash>>()>(context), std::span{metadata::base_table_t::originalLabels});
   if (originalTable->num_rows() == 0) {
-    originalTable = makeEmptyTable<metadata::base_table_t::ref>();
+    originalTable = makeEmptyTable("EMPTY", typename metadata::base_table_t::persistent_columns_t{});
   }
   if (defines.inputSchema == nullptr) {
     defines.inputSchema = originalTable->schema();

Framework/Core/include/Framework/AnalysisManagers.h

@@ -21,11 +21,11 @@
 namespace o2::framework
 {
 static constexpr std::array<header::DataOrigin, 4> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};
-static constexpr std::array<header::DataOrigin, 6> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}, header::DataOrigin{"EMB"}};
-static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
+// static constexpr std::array<header::DataOrigin, 6> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};
+// static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}};
 
 class DataOutputDirector;
-class ConfigContext;
+struct ConfigContext;
 
 // Helper class to be moved in the AnalysisSupport plugin at some point
 struct AnalysisSupportHelpers {

Framework/Core/include/Framework/AnalysisSupportHelpers.h

@@ -12,12 +12,12 @@
 
 namespace o2::soa
 {
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::BCs, aod::Timestamps>;
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksExtra>;
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksCov, aod::TracksExtra>;
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::FwdTracks, aod::FwdTracksCov>;
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run2MatchedSparse>;
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run3MatchedSparse>;
+template struct Join<aod::BCs, aod::Timestamps>;
+template struct Join<aod::Tracks, aod::TracksExtra>;
+template struct Join<aod::Tracks, aod::TracksCov, aod::TracksExtra>;
+template struct Join<aod::FwdTracks, aod::FwdTracksCov>;
+template struct Join<aod::Collisions, aod::Run2MatchedSparse>;
+template struct Join<aod::Collisions, aod::Run3MatchedSparse>;
 
-template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::TracksExtension, aod::StoredTracks>;
+template struct Join<aod::TracksExtension, aod::StoredTracks>;
 } // namespace o2::soa

Framework/Core/src/AnalysisDataModel.cxx

@@ -102,7 +102,7 @@ std::shared_ptr<DataOutputDirector> AnalysisSupportHelpers::getDataOutputDirecto
         // use the dangling outputs
         std::vector<InputSpec> danglingOutputs;
         for (auto ii = 0u; ii < OutputsInputs.size(); ii++) {
-          if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) {
+          if (DataSpecUtils::partialMatch(OutputsInputs[ii], AODOrigins) && isDangling[ii]) {
             danglingOutputs.emplace_back(OutputsInputs[ii]);
           }
         }