[CTF] Fix 0 Bit Packing

A bug in `ctf::EncodedBlocks` caused data to be packed incorrectly. The
problem occurs for sequences where all input values are identical, i.e.
for each index i in an array, the value of i = v. In broken CTFs,
metadata entries for `opt` is `PACK`,`alphabetRangeBits`, `min` and
`max` are `0`. The payload is a single word, containing a value.
During unpacking, no data is written into the decode buffer.

* For incorrectly written CTFs the original data can be restored by
writing the payload buffer `metadata.messageLength` times into the
decodeBuffer.
* For new CTFs where all input values are identical, no payload is
written. Instead `metadata.alphabetRangeBits` is set to 0 and
`metadata.min` contains the repeating value.
* When unpacking this special case, `metadata.min` is written
`messageLength` times.

The `rans::pack` and `rans::unpack` functions contain additional guards
that will throw a `PackingError`, in case it is requested to unpack 0
Bit wide blocks or pack data into  0 Bit wide blocks.

(cherry picked from commit bb1d6da)

Author: MichaelLettrich

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h

@@ -284,8 +284,10 @@ struct Block {
   // resize block and free up unused buffer space.
   void realignBlock()
   {
-    size_t sz = estimateSize(getNStored());
-    registry->offsFreeStart = (reinterpret_cast<char*>(payload) - registry->head) + sz;
+    if (payload) {
+      size_t sz = estimateSize(getNStored());
+      registry->offsFreeStart = (reinterpret_cast<char*>(payload) - registry->head) + sz;
+    }
   }
 
   /// store binary blob data (buffer filled from head to tail)
@@ -608,13 +610,17 @@ class EncodedBlocks
     using source_type = typename std::iterator_traits<input_IT>::value_type;
 
     rans::Metrics<source_type> metrics{};
+    metrics.getDatasetProperties().numSamples = std::distance(srcBegin, srcEnd);
 
-    const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd);
-    if (minIter != maxIter) {
+    if (metrics.getDatasetProperties().numSamples != 0) {
+      const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd);
       metrics.getDatasetProperties().min = *minIter;
       metrics.getDatasetProperties().max = *maxIter;
-      metrics.getDatasetProperties().alphabetRangeBits = rans::utils::getRangeBits(metrics.getDatasetProperties().min,
-                                                                                   metrics.getDatasetProperties().max);
+
+      // special case: if min === max, the range is 0 and the data can be reconstructed just via the metadata.
+      metrics.getDatasetProperties().alphabetRangeBits =
+        rans::utils::getRangeBits(metrics.getDatasetProperties().min,
+                                  metrics.getDatasetProperties().max);
     }
 
     return pack(srcBegin, srcEnd, slot, metrics, buffer);
@@ -638,7 +644,7 @@ class EncodedBlocks
   CTFIOSize decodeCopyImpl(dst_IT dest, int slot) const;
 
   ClassDefNV(EncodedBlocks, 3);
-};
+}; // namespace ctf
 
 ///_____________________________________________________________________________
 /// read from tree to non-flat object
@@ -877,7 +883,6 @@ inline auto EncodedBlocks<H, N, W>::create(VD& v)
 template <typename H, int N, typename W>
 void EncodedBlocks<H, N, W>::print(const std::string& prefix, int verbosity) const
 {
-  verbosity = 5;
   if (verbosity > 0) {
     LOG(info) << prefix << "Container of " << N << " blocks, size: " << size() << " bytes, unused: " << getFreeSize();
     for (int i = 0; i < N; i++) {
@@ -922,22 +927,26 @@ CTFIOSize EncodedBlocks<H, N, W>::decode(D_IT dest,                        // it
   const auto& ansVersion = getANSHeader();
   const auto& block = mBlocks[slot];
   const auto& md = mMetadata[slot];
-
-  if (!block.getNStored()) {
-    return {0, md.getUncompressedSize(), md.getCompressedSize()};
-  }
+  LOGP(debug, "Slot{} | NStored={} Ndict={} nData={}, MD: messageLength:{} opt:{} min:{} max:{} offs:{} width:{} ", slot, block.getNStored(), block.getNDict(), block.getNData(), md.messageLength, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth);
 
   if (ansVersion == ANSVersionCompat) {
+    if (!block.getNStored()) {
+      return {0, md.getUncompressedSize(), md.getCompressedSize()};
+    }
     if (md.opt == Metadata::OptStore::EENCODE) {
       return decodeCompatImpl(dest, slot, decoderExt);
     } else {
       return decodeCopyImpl(dest, slot);
     }
   } else if (ansVersion == ANSVersion1) {
+    if (md.opt == Metadata::OptStore::PACK) {
+      return decodeUnpackImpl(dest, slot);
+    }
+    if (!block.getNStored()) {
+      return {0, md.getUncompressedSize(), md.getCompressedSize()};
+    }
     if (md.opt == Metadata::OptStore::EENCODE) {
       return decodeRansV1Impl(dest, slot, decoderExt);
-    } else if (md.opt == Metadata::OptStore::PACK) {
-      return decodeUnpackImpl(dest, slot);
     } else {
       return decodeCopyImpl(dest, slot);
     }
@@ -1048,9 +1057,26 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeUnpackImpl(dst_IT dest, int slot) const
   const auto& block = mBlocks[slot];
   const auto& md = mMetadata[slot];
 
+  const size_t messageLength = md.messageLength;
   const size_t packingWidth = md.probabilityBits;
   const dest_t offset = md.min;
-  rans::unpack(block.getData(), md.messageLength, dest, packingWidth, offset);
+  const auto* srcIt = block.getData();
+  // we have a vector of one and the same value. All information is in the metadata
+  if (packingWidth == 0) {
+    const dest_t value = [&]() {
+      // Bugfix: We tried packing values with a width of 0 Bits;
+      if (md.nDataWords > 0) {
+        return static_cast<dest_t>(*srcIt);
+      }
+      // normal case:
+      return offset;
+    }();
+    for (size_t i = 0; i < messageLength; ++i) {
+      *dest++ = value;
+    }
+  } else {
+    rans::unpack(srcIt, messageLength, dest, packingWidth, offset);
+  }
   return {0, md.getUncompressedSize(), md.getCompressedSize()};
 };
 
@@ -1440,20 +1466,29 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::pack(const input_IT srcBegin, const i
   using storageBuffer_t = W;
   using input_t = typename std::iterator_traits<input_IT>::value_type;
 
-  const size_t messageLength = std::distance(srcBegin, srcEnd);
-
-  internal::Packer<input_t> packer{metrics};
-  size_t packingBufferWords = packer.template getPackingBufferSize<storageBuffer_t>(messageLength);
-  auto [thisBlock, thisMetadata] = expandStorage(slot, packingBufferWords, buffer);
+  const size_t messageLength = metrics.getDatasetProperties().numSamples;
+  const auto alphabetRangeBits = metrics.getDatasetProperties().alphabetRangeBits;
 
-  auto packedMessageEnd = packer.pack(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock());
-  const size_t packeSize = std::distance(thisBlock->getCreateData(), packedMessageEnd);
-  thisBlock->setNData(packeSize);
-  thisBlock->realignBlock();
+  auto* thisBlock = &mBlocks[slot];
+  auto* thisMetadata = &mMetadata[slot];
+  size_t packedSize = 0;
 
-  LOGP(info, "StoreData {} bytes, offs: {}:{}", packeSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packeSize * sizeof(storageBuffer_t));
+  if (messageLength == 0) {
+    *thisMetadata = detail::makeMetadataPack<input_t>(0, 0, 0, 0);
+  } else if (metrics.getDatasetProperties().alphabetRangeBits == 0) {
+    *thisMetadata = detail::makeMetadataPack<input_t>(messageLength, 0, *srcBegin, 0);
+  } else {
+    internal::Packer<input_t> packer{metrics};
+    size_t packingBufferWords = packer.template getPackingBufferSize<storageBuffer_t>(messageLength);
+    std::tie(thisBlock, thisMetadata) = expandStorage(slot, packingBufferWords, buffer);
+    auto packedMessageEnd = packer.pack(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock());
+    packedSize = std::distance(thisBlock->getCreateData(), packedMessageEnd);
+    *thisMetadata = detail::makeMetadataPack<input_t>(messageLength, packer.getPackingWidth(), packer.getOffset(), packedSize);
+    thisBlock->setNData(packedSize);
+    thisBlock->realignBlock();
+  }
 
-  *thisMetadata = detail::makeMetadataPack<input_t>(messageLength, packer.getPackingWidth(), packer.getOffset(), packeSize);
+  LOGP(info, "StoreData {} bytes, offs: {}:{}", packedSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packedSize * sizeof(storageBuffer_t));
   return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
 };
 

Utilities/rANS/include/rANS/internal/common/exceptions.h

@@ -92,6 +92,11 @@ class DecodingError : public CodingError
   using CodingError::CodingError;
 };
 
+class PackingError : public CodingError
+{
+  using CodingError::CodingError;
+};
+
 } // namespace o2::rans
 
 #endif /* RANS_INTERNAL_COMMON_EXCEPTIONS_ */

Utilities/rANS/include/rANS/internal/pack/pack.h

@@ -202,6 +202,13 @@ inline constexpr BitPtr pack(const input_T* __restrict inputBegin, size_t extent
   assert(outputBegin != nullptr);
 
   switch (packingWidth) {
+    case 0:
+      if (extent > 0) {
+        throw PackingError("Cannot pack data into 0 Bit wide blocks");
+      } else {
+        return BitPtr(outputBegin);
+      }
+      break;
     case 1:
       return packStreamImpl<input_T, output_T, 1>(inputBegin, extent, outputBegin, offset);
       break;
@@ -342,6 +349,15 @@ inline void unpack(const input_T* __restrict inputBegin, size_t extent, output_I
   using namespace utils;
   using dst_type = typename std::iterator_traits<output_IT>::value_type;
 
+  // cannot unpack into 0 bits
+  if (packingWidth == 0) {
+    if (extent == 0) {
+      return;
+    } else {
+      throw PackingError("Cannot unpack into 0 Bit wide data");
+    }
+  }
+
   auto unpackImpl = [&](auto packer) {
     output_IT outputIt = outputBegin;
     BitPtr iter{inputBegin};