WIP DPL Analysis: support BinaryView

The idea is to be able to have BinaryViews on top of the CCDB object
blobs which are already cached in shared memory, so that we can have
a simple table with rows of the kind:

(timestamp, blob-requested-ccdb-object-2, blob-requested-ccdb-object-2)

which then can be joined via the timestamp to provide access to the requested CCDB
Object.

Filling of the table via TableBuilder seems to work, retrieving the blobs currently crashes.

Author: ktf

Framework/Core/include/Framework/ASoA.h

@@ -23,11 +23,12 @@
 #include "Framework/ArrowTableSlicingCache.h" // IWYU pragma: export
 #include "Framework/SliceCache.h"             // IWYU pragma: export
 #include "Framework/VariantHelpers.h"         // IWYU pragma: export
-#include <arrow/table.h>                      // IWYU pragma: export
-#include <arrow/array.h>                      // IWYU pragma: export
-#include <arrow/util/config.h>                // IWYU pragma: export
-#include <gandiva/selection_vector.h>         // IWYU pragma: export
-#include <array>                              // IWYU pragma: export
+#include <arrow/array/array_binary.h>
+#include <arrow/table.h>              // IWYU pragma: export
+#include <arrow/array.h>              // IWYU pragma: export
+#include <arrow/util/config.h>        // IWYU pragma: export
+#include <gandiva/selection_vector.h> // IWYU pragma: export
+#include <array>                      // IWYU pragma: export
 #include <cassert>
 #include <fmt/format.h>
 #include <concepts>
@@ -579,7 +580,15 @@ class ColumnIterator : ChunkingPolicy
   }
 
   decltype(auto) operator*() const
-    requires((!std::same_as<bool, std::decay_t<T>>) && !std::same_as<arrow_array_for_t<T>, arrow::ListArray>)
+    requires((!std::same_as<bool, std::decay_t<T>>) && std::same_as<arrow_array_for_t<T>, arrow::BinaryViewArray>)
+  {
+    checkSkipChunk();
+    auto array = std::static_pointer_cast<arrow::BinaryViewArray>(mColumn->chunk(mCurrentChunk));
+    return array->GetView(*mCurrentPos - mFirstIndex);
+  }
+
+  decltype(auto) operator*() const
+    requires((!std::same_as<bool, std::decay_t<T>>) && !std::same_as<arrow_array_for_t<T>, arrow::ListArray> && !std::same_as<arrow_array_for_t<T>, arrow::BinaryViewArray>)
   {
     checkSkipChunk();
     return *(mCurrent + (*mCurrentPos >> SCALE_FACTOR));

Framework/Core/include/Framework/ArrowTypes.h

@@ -12,6 +12,7 @@
 #ifndef O2_FRAMEWORK_ARROWTYPES_H
 #define O2_FRAMEWORK_ARROWTYPES_H
 #include "arrow/type_fwd.h"
+#include <span>
 
 namespace o2::soa
 {
@@ -62,6 +63,10 @@ template <>
 struct arrow_array_for<double> {
   using type = arrow::DoubleArray;
 };
+template <>
+struct arrow_array_for<std::span<std::byte>> {
+  using type = arrow::BinaryViewArray;
+};
 template <int N>
 struct arrow_array_for<float[N]> {
   using type = arrow::FixedSizeListArray;

Framework/Core/include/Framework/TableBuilder.h

@@ -98,6 +98,7 @@ O2_ARROW_STL_CONVERSION(long unsigned, UInt64Type)
 O2_ARROW_STL_CONVERSION(float, FloatType)
 O2_ARROW_STL_CONVERSION(double, DoubleType)
 O2_ARROW_STL_CONVERSION(std::string, StringType)
+O2_ARROW_STL_CONVERSION(std::span<std::byte>, BinaryViewType)
 } // namespace detail
 
 void addLabelToSchema(std::shared_ptr<arrow::Schema>& schema, const char* label);
@@ -274,6 +275,29 @@ struct BuilderMaker<bool> {
   }
 };
 
+template <>
+struct BuilderMaker<std::span<std::byte>> {
+  using FillType = std::span<std::byte>;
+  using STLValueType = std::span<std::byte>;
+  using ArrowType = typename detail::ConversionTraits<std::span<std::byte>>::ArrowType;
+  using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
+
+  static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
+  {
+    return std::make_unique<BuilderType>(pool);
+  }
+
+  static std::shared_ptr<arrow::DataType> make_datatype()
+  {
+    return arrow::TypeTraits<ArrowType>::type_singleton();
+  }
+
+  static arrow::Status append(BuilderType& builder, std::span<std::byte> value)
+  {
+    return builder.Append((char*)value.data(), (int64_t)value.size());
+  }
+};
+
 template <typename ITERATOR>
 struct BuilderMaker<std::pair<ITERATOR, ITERATOR>> {
   using FillType = std::pair<ITERATOR, ITERATOR>;
@@ -422,6 +446,13 @@ struct DirectInsertion {
     return builder->Append(value);
   }
 
+  template <typename BUILDER>
+    requires std::same_as<std::span<std::byte>, T>
+  arrow::Status append(BUILDER& builder, T value)
+  {
+    return builder->Append((char*)value.data(), (int64_t)value.size());
+  }
+
   template <typename BUILDER>
   arrow::Status flush(BUILDER&)
   {
@@ -569,7 +600,7 @@ template <typename... ARGS>
 using IndexedHoldersTuple = decltype(makeHolderTypes<ARGS...>());
 
 template <typename T>
-concept ShouldNotDeconstruct = std::is_bounded_array_v<T> || std::is_arithmetic_v<T> || framework::is_base_of_template_v<std::vector, T>;
+concept ShouldNotDeconstruct = std::is_bounded_array_v<T> || std::is_arithmetic_v<T> || framework::is_base_of_template_v<std::vector, T> || std::same_as<std::span<std::byte>, T>;
 
 /// Helper class which creates a lambda suitable for building
 /// an arrow table from a tuple. This can be used, for example

Framework/Core/test/test_TableBuilder.cxx

@@ -1,4 +1,4 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// Copyright 2019-2025 CERN and copyright holders of ALICE O2.
 // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
 // All rights not expressly granted are reserved.
 //
@@ -19,6 +19,8 @@
 #include <arrow/ipc/writer.h>
 #include <arrow/ipc/reader.h>
 
+#include <string_view>
+
 using namespace o2::framework;
 
 // We use a different namespace to avoid clashes with the
@@ -27,10 +29,12 @@ namespace test2
 {
 DECLARE_SOA_COLUMN_FULL(X, x, uint64_t, "x");
 DECLARE_SOA_COLUMN_FULL(Y, y, uint64_t, "y");
+DECLARE_SOA_COLUMN_FULL(Blob, blob, std::span<std::byte>, "blob");
 DECLARE_SOA_COLUMN_FULL(Pos, pos, int[4], "pos");
 } // namespace test2
 
 using TestTable = o2::soa::InPlaceTable<0, test2::X, test2::Y>;
+using SpanTable = o2::soa::InPlaceTable<0, test2::Blob>;
 using ArrayTable = o2::soa::InPlaceTable<0, test2::Pos>;
 
 TEST_CASE("TestTableBuilder")
@@ -189,6 +193,35 @@ TEST_CASE("TestTableBuilderMore")
   REQUIRE(table->schema()->field(3)->type()->id() == arrow::boolean()->id());
 }
 
+TEST_CASE("TestSpan")
+{
+  TableBuilder builder;
+  std::vector<std::byte> buffer{10, std::byte{'c'}};
+  std::vector<std::byte> buffer1{10, std::byte{'a'}};
+
+  auto rowWriter = builder.persist<std::span<std::byte>>({"blob"});
+  rowWriter(0, std::span(buffer));
+  rowWriter(0, std::span(buffer.data() + 1, 9));
+  rowWriter(0, std::span(buffer1.data(), 3));
+  rowWriter(0, std::span(buffer1.data(), 1));
+  auto table = builder.finalize();
+
+  REQUIRE(table->num_columns() == 1);
+  REQUIRE(table->num_rows() == 4);
+  REQUIRE(table->schema()->field(0)->name() == "blob");
+  REQUIRE(table->schema()->field(0)->type()->id() == arrow::binary_view()->id());
+
+  auto readBack = SpanTable{table};
+  auto row = readBack.begin();
+  REQUIRE(row.blob() == "cccccccccc");
+  ++row;
+  REQUIRE(row.blob() == "ccccccccc");
+  ++row;
+  REQUIRE(row.blob() == "aaa");
+  ++row;
+  REQUIRE(row.blob() == "a");
+}
+
 TEST_CASE("TestSoAIntegration")
 {
   TableBuilder builder;