DPL: drop obsolete TreeToTable code

Now using the arrow::Dataset API.

Author: ktf

Framework/Core/CMakeLists.txt

@@ -250,7 +250,6 @@ add_executable(o2-test-framework-core
               test/test_Variants.cxx
               test/test_WorkflowHelpers.cxx
               test/test_WorkflowSerialization.cxx
-              test/test_TreeToTable.cxx
               test/test_DataOutputDirector.cxx
               test/unittest_SimpleOptionsRetriever.cxx
               test/unittest_DataSpecUtils.cxx
@@ -348,7 +347,6 @@ foreach(b
         EventMixing
         HistogramRegistry
         TableToTree
-        TreeToTable
         ExternalFairMQDeviceProxies
         )
   o2_add_executable(benchmark-${b}

Framework/Core/include/Framework/DataAllocator.h

@@ -233,15 +233,6 @@ class DataAllocator
     return tb;
   }
 
-  template <typename T, typename... Args>
-    requires(requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); })
-  decltype(auto) make(const Output& spec, Args... args)
-  {
-    auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
-    adopt(spec, t2t);
-    return t2t;
-  }
-
   template <typename T, typename... Args>
     requires(requires { static_cast<struct FragmentToBatch>(std::declval<std::decay_t<T>>()); })
   decltype(auto) make(const Output& spec, Args... args)
@@ -288,11 +279,6 @@ class DataAllocator
   void
     adopt(const Output& spec, LifetimeHolder<struct TableBuilder>&);
 
-  /// Adopt a Tree2Table in the framework and serialise / send
-  /// it as an Arrow table to all consumers of @a spec once done
-  void
-    adopt(const Output& spec, LifetimeHolder<struct TreeToTable>&);
-
   /// Adopt a Source2Batch in the framework and serialise / send
   /// it as an Arrow Dataset to all consumers of @a spec once done
   void

Framework/Core/include/Framework/TableTreeHelpers.h

@@ -36,19 +36,6 @@ namespace o2::framework
 //    OR t2t.addBranch(column.get(), field.get()), ...;
 //  . t2t.process();
 //
-// .............................................................................
-// -----------------------------------------------------------------------------
-// TreeToTable allows to fill the contents of a given TTree to an arrow::Table
-//  ColumnIterator is used by TreeToTable
-//
-// To copy the contents of a tree tr to a table ta do:
-//  . TreeToTable t2t(tr);
-//  . t2t.addColumn(columnname1); t2t.addColumn(columnname2); ...
-//    OR
-//    t2t.addAllColumns();
-//  . auto ta = t2t.process();
-//
-// .............................................................................
 struct ROOTTypeInfo {
   EDataType type;
   char suffix[3];
@@ -127,24 +114,6 @@ class TableToTree
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
 
-class TreeToTable
-{
- public:
-  TreeToTable(arrow::MemoryPool* pool = arrow::default_memory_pool());
-  void setLabel(const char* label);
-  void addAllColumns(TTree* tree, std::vector<std::string>&& names = {});
-  void fill(TTree*);
-  std::shared_ptr<arrow::Table> finalize();
-
- private:
-  arrow::MemoryPool* mArrowMemoryPool;
-  std::vector<std::unique_ptr<BranchToColumn>> mBranchReaders;
-  std::string mTableLabel;
-  std::shared_ptr<arrow::Table> mTable;
-
-  void addReader(TBranch* branch, std::string const& name, bool VLA);
-};
-
 class FragmentToBatch
 {
  public:

Framework/Core/src/DataAllocator.cxx

@@ -241,38 +241,6 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
   context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex);
 }
 
-void DataAllocator::adopt(const Output& spec, LifetimeHolder<TreeToTable>& t2t)
-{
-  auto& timingInfo = mRegistry.get<TimingInfo>();
-  RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-
-  auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
-  auto& context = mRegistry.get<ArrowContext>();
-
-  auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {
-    return transport->CreateMessage(s);
-  };
-  auto buffer = std::make_shared<FairMQResizableBuffer>(creator);
-
-  t2t.callback = [buffer = buffer, transport = context.proxy().getOutputTransport(routeIndex)](TreeToTable& tree) {
-    // Serialization happens in here, so that we can
-    // get rid of the intermediate tree 2 table object, saving memory.
-    auto table = tree.finalize();
-    doWriteTable(buffer, table.get());
-    // deletion happens in the caller
-  };
-
-  /// To finalise this we write the table to the buffer.
-  /// FIXME: most likely not a great idea. We should probably write to the buffer
-  ///        directly in the TableBuilder, incrementally.
-  auto finalizer = [](std::shared_ptr<FairMQResizableBuffer> b) -> void {
-    // This is empty because we already serialised the object when
-    // the LifetimeHolder goes out of scope.
-  };
-
-  context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex);
-}
-
 void DataAllocator::adopt(const Output& spec, LifetimeHolder<FragmentToBatch>& f2b)
 {
   auto& timingInfo = mRegistry.get<TimingInfo>();

Framework/Core/src/TableTreeHelpers.cxx

@@ -447,11 +447,6 @@ std::shared_ptr<TTree> TableToTree::process()
   return mTree;
 }
 
-TreeToTable::TreeToTable(arrow::MemoryPool* pool)
-  : mArrowMemoryPool{pool}
-{
-}
-
 namespace
 {
 struct BranchInfo {
@@ -461,113 +456,6 @@ struct BranchInfo {
 };
 } // namespace
 
-void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
-{
-  auto branches = tree->GetListOfBranches();
-  auto n = branches->GetEntries();
-  if (n == 0) {
-    throw runtime_error("Tree has no branches");
-  }
-
-  std::vector<BranchInfo> branchInfos;
-  for (auto i = 0; i < n; ++i) {
-    auto branch = static_cast<TBranch*>(branches->At(i));
-    auto name = std::string{branch->GetName()};
-    auto pos = name.find(TableTreeHelpers::sizeBranchSuffix);
-    if (pos != std::string::npos) {
-      name.erase(pos);
-      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
-    } else {
-      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
-        return bi.name == name;
-      });
-      if (lookup == branchInfos.end()) {
-        branchInfos.emplace_back(BranchInfo{name, branch, false});
-      } else {
-        lookup->ptr = branch;
-      }
-    }
-  }
-
-  if (names.empty()) {
-    for (auto& bi : branchInfos) {
-      addReader(bi.ptr, bi.name, bi.mVLA);
-    }
-  } else {
-    for (auto& name : names) {
-      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
-        return name == bi.name;
-      });
-      if (lookup != branchInfos.end()) {
-        addReader(lookup->ptr, lookup->name, lookup->mVLA);
-      }
-    }
-    if (names.size() != mBranchReaders.size()) {
-      LOGF(warn, "Not all requested columns were found in the tree");
-    }
-  }
-  if (mBranchReaders.empty()) {
-    throw runtime_error("No columns will be read");
-  }
-  // Was affected by https://github.com/root-project/root/issues/8962
-  // Re-enabling this seems to cut the number of IOPS in half
-  tree->SetCacheSize(25000000);
-  // tree->SetClusterPrefetch(true);
-  for (auto& reader : mBranchReaders) {
-    tree->AddBranchToCache(reader->branch());
-    if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
-      std::string sizeBranchName = reader->branch()->GetName();
-      sizeBranchName += "_size";
-      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
-      if (sizeBranch) {
-        tree->AddBranchToCache(sizeBranch);
-      }
-    }
-  }
-  tree->StopCacheLearningPhase();
-}
-
-void TreeToTable::setLabel(const char* label)
-{
-  mTableLabel = label;
-}
-
-void TreeToTable::fill(TTree* tree)
-{
-  std::vector<std::shared_ptr<arrow::ChunkedArray>> columns;
-  std::vector<std::shared_ptr<arrow::Field>> fields;
-  static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
-  O2_SIGNPOST_ID_FROM_POINTER(sid, tabletree_helpers, &buffer);
-  O2_SIGNPOST_START(tabletree_helpers, sid, "TreeToTable", "Filling %{public}s", tree->GetName());
-  for (auto& reader : mBranchReaders) {
-    buffer.Reset();
-    auto arrayAndField = reader->read(&buffer);
-    columns.push_back(arrayAndField.first);
-    fields.push_back(arrayAndField.second);
-  }
-  O2_SIGNPOST_END(tabletree_helpers, sid, "TreeToTable", "Done filling.");
-
-  auto schema = std::make_shared<arrow::Schema>(fields, std::make_shared<arrow::KeyValueMetadata>(std::vector{std::string{"label"}}, std::vector{mTableLabel}));
-  mTable = arrow::Table::Make(schema, columns);
-}
-
-void TreeToTable::addReader(TBranch* branch, std::string const& name, bool VLA)
-{
-  static TClass* cls;
-  EDataType type;
-  branch->GetExpectedType(cls, type);
-  auto listSize = -1;
-  if (!VLA) {
-    listSize = static_cast<TLeaf*>(branch->GetListOfLeaves()->At(0))->GetLenStatic();
-  }
-  mBranchReaders.emplace_back(std::make_unique<BranchToColumn>(branch, VLA, name, type, listSize, mArrowMemoryPool));
-}
-
-std::shared_ptr<arrow::Table> TreeToTable::finalize()
-{
-  return mTable;
-}
-
 FragmentToBatch::FragmentToBatch(StreamerCreator creator, std::shared_ptr<arrow::dataset::FileFragment> fragment, arrow::MemoryPool* pool)
   : mFragment{std::move(fragment)},
     mArrowMemoryPool{pool},