Added delta+zigzag+varint compression, added drawCMV.C macro for visualization

Author: tubagundem

Detectors/TPC/calibration/include/TPCCalibration/CMVContainer.h

@@ -18,8 +18,8 @@
 
 #include <string>
 #include <memory>
-#include <stdexcept>
-#include <fmt/format.h>
+#include <vector>
+#include <cstdint>
 
 #include "TTree.h"
 #include "TPCBase/CRU.h"
@@ -28,6 +28,31 @@
 namespace o2::tpc
 {
 
+struct CMVPerTF; // forward declaration
+
+/// Delta+zigzag+varint compressed CMV data for one TF across all CRUs
+/// Produced by CMVPerTF::compress(), restored with decompress()
+/// Each TTree entry corresponds to one CMVPerTFCompressed object (one TF)
+struct CMVPerTFCompressed {
+  uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
+  uint16_t firstBC{0};    ///< First bunch crossing of this TF (copied from CMVPerTF)
+
+  /// Delta+zigzag+varint encoded CMV values
+  /// Layout: CRU-major, time-minor; delta is reset to zero at each CRU boundary
+  std::vector<uint8_t> mCompressedData;
+
+  /// Restore a CMVPerTF from this compressed object into *cmv (must not be null)
+  void decompress(CMVPerTF* cmv) const;
+
+ private:
+  static uint16_t signedToCmv(int32_t val);                               ///< Signed integer -> sign-magnitude uint16_t
+  static int32_t zigzagDecode(uint32_t value);                            ///< Zigzag decode
+  static uint32_t decodeVarint(const uint8_t*& data, const uint8_t* end); ///< Varint decode
+
+ public:
+  ClassDefNV(CMVPerTFCompressed, 1)
+};
+
 /// CMV data for one TF across all CRUs
 /// Raw 16-bit CMV values are stored in a flat C array indexed as [cru * NTimeBinsPerTF + timeBin]
 /// CRU::MaxCRU and cmv::NTimeBinsPerTF are compile-time constants, so no dynamic allocation is needed
@@ -40,33 +65,27 @@ struct CMVPerTF {
   uint16_t mDataPerTF[CRU::MaxCRU * cmv::NTimeBinsPerTF]{};
 
   /// Return the raw 16-bit CMV value for a given CRU and timebin within this TF
-  uint16_t getCMV(const int cru, const int timeBin) const
-  {
-    if (cru < 0 || cru >= static_cast<int>(CRU::MaxCRU)) {
-      throw std::out_of_range(fmt::format("CMVPerTF::getCMV: cru {} out of range [0, {})", cru, static_cast<int>(CRU::MaxCRU)));
-    }
-    if (timeBin < 0 || static_cast<uint32_t>(timeBin) >= cmv::NTimeBinsPerTF) {
-      throw std::out_of_range(fmt::format("CMVPerTF::getCMV: timeBin {} out of range [0, {})", timeBin, static_cast<int>(cmv::NTimeBinsPerTF)));
-    }
-    return mDataPerTF[cru * cmv::NTimeBinsPerTF + timeBin];
-  }
+  uint16_t getCMV(const int cru, const int timeBin) const;
 
   /// Return the float CMV value for a given CRU and timebin within this TF
-  float getCMVFloat(const int cru, const int timeBin) const
-  {
-    auto cmv = getCMV(cru, timeBin);
-    const bool positive = (cmv >> 15) & 1;          // bit 15: sign (1=positive, 0=negative)
-    const float magnitude = (cmv & 0x7FFF) / 128.f; // lower 15 bits, shift right by 7 (divide by 2^7)
-    return positive ? magnitude : -magnitude;
-  }
+  float getCMVFloat(const int cru, const int timeBin) const;
+
+  /// Compress this object into a CMVPerTFCompressed using delta+zigzag+varint encoding
+  CMVPerTFCompressed compress() const;
 
   /// Serialise into a TTree; each Fill() call appends one entry (one TF)
   std::unique_ptr<TTree> toTTree() const;
 
   /// Write the TTree to a ROOT file
-  void writeToFile(const std::string& filename, const std::unique_ptr<TTree>& tree) const;
+  static void writeToFile(const std::string& filename, const std::unique_ptr<TTree>& tree);
+
+ private:
+  static int32_t cmvToSigned(uint16_t raw);                                ///< Sign-magnitude uint16_t → signed integer
+  static uint32_t zigzagEncode(int32_t value);                             ///< Zigzag encode
+  static void encodeVarintInto(uint32_t value, std::vector<uint8_t>& out); ///< Varint encode
 
-  ClassDefNV(CMVPerTF, 8)
+ public:
+  ClassDefNV(CMVPerTF, 1)
 };
 
 } // namespace o2::tpc

Detectors/TPC/calibration/macro/drawCMV.C

@@ -0,0 +1,126 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include <string>
+#include <string_view>
+#include <fmt/format.h>
+
+#include "TFile.h"
+#include "TParameter.h"
+#include "TTree.h"
+#include "TH2F.h"
+#include "TCanvas.h"
+
+#include "TPCCalibration/CMVContainer.h"
+#include "TPCBase/Utils.h"
+#endif
+
+using namespace o2::tpc;
+
+/// Draw CMV (Common Mode Values) vs timebin from a CCDB TTree file
+/// \param filename  input ROOT file containing the ccdb_object TTree
+/// \param outDir    output directory for saved plots; nothing is saved if empty
+/// \return          array of canvases
+TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
+{
+  TObjArray* arrCanvases = new TObjArray;
+  arrCanvases->SetName("CMV");
+
+  // open file
+  TFile f(filename.data(), "READ");
+  if (f.IsZombie()) {
+    fmt::print("ERROR: cannot open '{}'\n", filename);
+    return arrCanvases;
+  }
+  fmt::print("Opened file: {}\n", filename);
+
+  // get TTree
+  TTree* tree = nullptr;
+  f.GetObject("ccdb_object", tree);
+  if (!tree) {
+    fmt::print("ERROR: TTree 'ccdb_object' not found\n");
+    return arrCanvases;
+  }
+  fmt::print("Tree 'ccdb_object' found, entries: {}\n", tree->GetEntries());
+
+  // read metadata
+  long firstTF = -1, lastTF = -1;
+  if (auto* userInfo = tree->GetUserInfo()) {
+    for (int i = 0; i < userInfo->GetSize(); ++i) {
+      if (auto* p = dynamic_cast<TParameter<long>*>(userInfo->At(i))) {
+        if (std::string(p->GetName()) == "firstTF")
+          firstTF = p->GetVal();
+        if (std::string(p->GetName()) == "lastTF")
+          lastTF = p->GetVal();
+      }
+    }
+  }
+  fmt::print("firstTF: {}, lastTF: {}\n", firstTF, lastTF);
+
+  const int nEntries = tree->GetEntries();
+  if (nEntries == 0) {
+    fmt::print("ERROR: no entries in tree\n");
+    return arrCanvases;
+  }
+
+  constexpr int nCRUs = CRU::MaxCRU;
+  constexpr int nTimeBins = cmv::NTimeBinsPerTF;
+
+  TH2F* h2d = new TH2F("hCMVvsTimeBin", ";Timebin (200 ns);Common Mode Values (ADC)",
+                       nTimeBins / 16, 0, nTimeBins,
+                       110, -100, 10);
+  h2d->SetStats(0);
+
+  // branch setup
+  o2::tpc::CMVPerTFCompressed* tfEntry = nullptr;
+  tree->SetBranchAddress("CMVPerTFCompressed", &tfEntry);
+
+  // allocate once outside the loop to avoid repeated zero-initialisation of the large array
+  auto* tf = new CMVPerTF();
+
+  long firstOrbit = -1;
+
+  for (int i = 0; i < nEntries; ++i) {
+    tree->GetEntry(i);
+    tfEntry->decompress(tf);
+
+    if (i == 0) {
+      firstOrbit = tf->firstOrbit;
+    }
+
+    for (int cru = 0; cru < nCRUs; ++cru) {
+      for (int tb = 0; tb < nTimeBins; ++tb) {
+        h2d->Fill(tb, tf->getCMVFloat(cru, tb));
+      }
+    }
+  }
+
+  delete tf;
+  tree->ResetBranchAddresses();
+  delete tfEntry;
+
+  fmt::print("firstOrbit: {}\n", firstOrbit);
+
+  // draw
+  auto* c = new TCanvas("cCMVvsTimeBin", "");
+  c->SetLogz();
+  h2d->Draw("colz");
+
+  arrCanvases->Add(c);
+
+  if (outDir.size()) {
+    utils::saveCanvases(*arrCanvases, outDir, "png,pdf", "CMVCanvases.root");
+  }
+
+  f.Close();
+  return arrCanvases;
+}

Detectors/TPC/calibration/src/CMVContainer.cxx

@@ -13,15 +13,125 @@
 /// @author Tuba Gündem, tuba.gundem@cern.ch
 
 #include <stdexcept>
+#include <cstdint>
+#include <cmath>
 #include <fmt/format.h>
 
 #include "TFile.h"
 
 #include "TPCCalibration/CMVContainer.h"
+#include "TPCBase/CRU.h"
+#include "DataFormatsTPC/CMV.h"
 
 namespace o2::tpc
 {
 
+int32_t CMVPerTF::cmvToSigned(uint16_t raw)
+{
+  const int32_t mag = raw & 0x7FFF;
+  return (raw >> 15) ? mag : -mag;
+}
+
+uint32_t CMVPerTF::zigzagEncode(int32_t value)
+{
+  return (static_cast<uint32_t>(value) << 1) ^ static_cast<uint32_t>(value >> 31);
+}
+
+void CMVPerTF::encodeVarintInto(uint32_t value, std::vector<uint8_t>& out)
+{
+  while (value > 0x7F) {
+    out.push_back(static_cast<uint8_t>((value & 0x7F) | 0x80));
+    value >>= 7;
+  }
+  out.push_back(static_cast<uint8_t>(value));
+}
+
+uint16_t CMVPerTFCompressed::signedToCmv(int32_t val)
+{
+  const uint16_t mag = static_cast<uint16_t>(std::abs(val)) & 0x7FFF;
+  return static_cast<uint16_t>((val >= 0 ? 0x8000u : 0u) | mag);
+}
+
+int32_t CMVPerTFCompressed::zigzagDecode(uint32_t value)
+{
+  return static_cast<int32_t>((value >> 1) ^ -(value & 1));
+}
+
+uint32_t CMVPerTFCompressed::decodeVarint(const uint8_t*& data, const uint8_t* end)
+{
+  uint32_t value = 0;
+  int shift = 0;
+  while (data < end && (*data & 0x80)) {
+    value |= static_cast<uint32_t>(*data & 0x7F) << shift;
+    shift += 7;
+    ++data;
+  }
+  if (data >= end) {
+    throw std::runtime_error("CMVPerTFCompressed::decompress: unexpected end of compressed data");
+  }
+  value |= static_cast<uint32_t>(*data) << shift;
+  ++data;
+  return value;
+}
+
+uint16_t CMVPerTF::getCMV(const int cru, const int timeBin) const
+{
+  if (cru < 0 || cru >= static_cast<int>(CRU::MaxCRU)) {
+    throw std::out_of_range(fmt::format("CMVPerTF::getCMV: cru {} out of range [0, {})", cru, static_cast<int>(CRU::MaxCRU)));
+  }
+  if (timeBin < 0 || static_cast<uint32_t>(timeBin) >= cmv::NTimeBinsPerTF) {
+    throw std::out_of_range(fmt::format("CMVPerTF::getCMV: timeBin {} out of range [0, {})", timeBin, static_cast<int>(cmv::NTimeBinsPerTF)));
+  }
+  return mDataPerTF[cru * cmv::NTimeBinsPerTF + timeBin];
+}
+
+float CMVPerTF::getCMVFloat(const int cru, const int timeBin) const
+{
+  const uint16_t raw = getCMV(cru, timeBin);
+  const bool positive = (raw >> 15) & 1;          // bit 15: sign (1=positive, 0=negative)
+  const float magnitude = (raw & 0x7FFF) / 128.f; // lower 15 bits, shift right by 7 (divide by 2^7)
+  return positive ? magnitude : -magnitude;
+}
+
+CMVPerTFCompressed CMVPerTF::compress() const
+{
+  CMVPerTFCompressed out;
+  out.firstOrbit = firstOrbit;
+  out.firstBC = firstBC;
+  out.mCompressedData.reserve(static_cast<size_t>(CRU::MaxCRU) * cmv::NTimeBinsPerTF);
+
+  for (int cru = 0; cru < static_cast<int>(CRU::MaxCRU); ++cru) {
+    int32_t prev = 0;
+    for (uint32_t tb = 0; tb < cmv::NTimeBinsPerTF; ++tb) {
+      const int32_t val = cmvToSigned(mDataPerTF[cru * cmv::NTimeBinsPerTF + tb]);
+      const int32_t delta = val - prev;
+      prev = val;
+      encodeVarintInto(zigzagEncode(delta), out.mCompressedData);
+    }
+  }
+  return out;
+}
+
+void CMVPerTFCompressed::decompress(CMVPerTF* cmv) const
+{
+  if (!cmv) {
+    throw std::invalid_argument("CMVPerTFCompressed::decompress: cmv pointer is null");
+  }
+  cmv->firstOrbit = firstOrbit;
+  cmv->firstBC = firstBC;
+  const uint8_t* ptr = mCompressedData.data();
+  const uint8_t* end = ptr + mCompressedData.size();
+
+  for (int cru = 0; cru < static_cast<int>(CRU::MaxCRU); ++cru) {
+    int32_t prev = 0;
+    for (uint32_t tb = 0; tb < cmv::NTimeBinsPerTF; ++tb) {
+      const int32_t val = prev + zigzagDecode(decodeVarint(ptr, end));
+      prev = val;
+      cmv->mDataPerTF[cru * cmv::NTimeBinsPerTF + tb] = signedToCmv(val);
+    }
+  }
+}
+
 std::unique_ptr<TTree> CMVPerTF::toTTree() const
 {
   auto tree = std::make_unique<TTree>("ccdb_object", "ccdb_object");
@@ -37,7 +147,7 @@ std::unique_ptr<TTree> CMVPerTF::toTTree() const
   return tree;
 }
 
-void CMVPerTF::writeToFile(const std::string& filename, const std::unique_ptr<TTree>& tree) const
+void CMVPerTF::writeToFile(const std::string& filename, const std::unique_ptr<TTree>& tree)
 {
   TFile f(filename.c_str(), "RECREATE");
   if (f.IsZombie()) {

Detectors/TPC/calibration/src/TPCCalibrationLinkDef.h

@@ -125,5 +125,6 @@
 #pragma link C++ class o2::tpc::PressureTemperatureHelper + ;
 
 #pragma link C++ class o2::tpc::CMVPerTF + ;
+#pragma link C++ class o2::tpc::CMVPerTFCompressed + ;
 
 #endif