DPL: use rapidjson to dump metrics

Author: ktf

Framework/Core/include/Framework/DeviceMetricsInfo.h

@@ -154,11 +154,11 @@ struct DeviceMetricsInfo {
   std::vector<MetricsStorage<StringMetric>> stringMetrics; // We do not keep so many strings as metrics as history is less relevant.
   std::vector<MetricsStorage<float>> floatMetrics;
   std::vector<MetricsStorage<int8_t>> enumMetrics;
-  std::vector<std::array<size_t, metricStorageSize<int>()>> intTimestamps;
-  std::vector<std::array<size_t, metricStorageSize<uint64_t>()>> uint64Timestamps;
-  std::vector<std::array<size_t, metricStorageSize<float>()>> floatTimestamps;
-  std::vector<std::array<size_t, metricStorageSize<StringMetric>()>> stringTimestamps;
-  std::vector<std::array<size_t, metricStorageSize<int8_t>()>> enumTimestamps;
+  std::vector<TimestampsStorage<int>> intTimestamps;
+  std::vector<TimestampsStorage<uint64_t>> uint64Timestamps;
+  std::vector<TimestampsStorage<float>> floatTimestamps;
+  std::vector<TimestampsStorage<StringMetric>> stringTimestamps;
+  std::vector<TimestampsStorage<int8_t>> enumTimestamps;
   std::vector<float> max;
   std::vector<float> min;
   std::vector<float> average;

Framework/Core/src/ResourcesMonitoringHelper.cxx

@@ -8,12 +8,13 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-
 #include "ResourcesMonitoringHelper.h"
 #include "Framework/DeviceMetricsInfo.h"
-#define BOOST_BIND_GLOBAL_PLACEHOLDERS
-#include <boost/property_tree/json_parser.hpp>
-#include <fstream>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/ostreamwrapper.h>
+
+#include <ostream>
+#include <string>
 #include <string_view>
 #include <algorithm>
 #include <cassert>
@@ -22,34 +23,28 @@
 using namespace o2::framework;
 
 template <typename T>
-inline static T retriveValue(T val)
+void fillNodeWithValue(rapidjson::Writer<rapidjson::OStreamWrapper>& w,
+                       size_t filledMetrics,
+                       MetricsStorage<T> const& metricsStorage,
+                       TimestampsStorage<T> const& timestampsStorage)
 {
-  return val;
-}
-
-inline static std::string retriveValue(const std::reference_wrapper<const StringMetric> val)
-{
-  return std::string(val.get().data);
-}
-
-template <typename T, typename TIMESTAMPS>
-boost::property_tree::ptree fillNodeWithValue(const DeviceMetricsInfo& deviceMetrics,
-                                              const T& metricsStorage, const TIMESTAMPS& timestampsStorage, size_t labelIndex, size_t storeIndex)
-{
-  unsigned int loopRange = std::min(deviceMetrics.metrics[labelIndex].filledMetrics, metricsStorage[storeIndex].size());
-  boost::property_tree::ptree metricNode;
+  unsigned int loopRange = std::min(filledMetrics, metricsStorage.size());
 
+  w.StartArray();
   for (unsigned int idx = 0; idx < loopRange; ++idx) {
-    boost::property_tree::ptree values;
-    values.add("timestamp", timestampsStorage[storeIndex][idx]);
+    w.StartObject();
+    w.Key("timestamp");
+    std::string s = std::to_string(timestampsStorage[idx]);
+    w.String(s.c_str(), s.size());
+    w.Key("value");
     if constexpr (std::is_arithmetic_v<T>) {
-      values.add("value", std::to_string(retriveValue(std::cref(metricsStorage[storeIndex][idx]))));
+      w.String(std::to_string(metricsStorage[idx]).c_str());
     } else {
-      values.add("value", retriveValue(std::cref(metricsStorage[storeIndex][idx])));
+      w.String(metricsStorage[idx].data);
     }
-    metricNode.push_back(std::make_pair("", values));
+    w.EndObject();
   }
-  return metricNode;
+  w.EndArray();
 }
 
 bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetricsInfo>& metrics,
@@ -58,19 +53,23 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric
                                                   std::vector<std::regex> const& performanceMetricsRegex,
                                                   std::ostream& out) noexcept
 {
-
   assert(metrics.size() == specs.size());
 
   if (metrics.empty()) {
     return false;
   }
 
-  boost::property_tree::ptree root;
-  for (unsigned int idx = 0; idx < metrics.size(); ++idx) {
+  rapidjson::OStreamWrapper osw(out);
+  rapidjson::PrettyWriter<rapidjson::OStreamWrapper> w(osw);
+
+  // Top level obejct for all the metrics
+  w.StartObject();
 
+  for (unsigned int idx = 0; idx < metrics.size(); ++idx) {
+    w.Key(specs[idx].id.c_str());
     const auto& deviceMetrics = metrics[idx];
-    boost::property_tree::ptree deviceRoot;
 
+    w.StartObject();
     for (size_t mi = 0; mi < deviceMetrics.metricLabels.size(); mi++) {
       std::string_view metricLabel{deviceMetrics.metricLabels[mi].label, deviceMetrics.metricLabels[mi].size};
 
@@ -83,40 +82,42 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric
       }
       auto storeIdx = deviceMetrics.metrics[mi].storeIdx;
 
+      size_t filledMetrics = deviceMetrics.metrics[mi].filledMetrics;
       if (deviceMetrics.metrics[mi].filledMetrics == 0) {
         continue;
       }
-      // if so
-
-      boost::property_tree::ptree metricNode;
-
+      w.Key(metricLabel.data(), metricLabel.size());
       switch (deviceMetrics.metrics[mi].type) {
         case MetricType::Int:
-          metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.intMetrics, deviceMetrics.intTimestamps, mi, storeIdx);
+          fillNodeWithValue(w, filledMetrics, deviceMetrics.intMetrics[storeIdx],
+                            deviceMetrics.intTimestamps[storeIdx]);
           break;
 
         case MetricType::Float:
-          metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.floatMetrics, deviceMetrics.floatTimestamps, mi, storeIdx);
+          fillNodeWithValue(w, filledMetrics, deviceMetrics.floatMetrics[storeIdx],
+                            deviceMetrics.floatTimestamps[storeIdx]);
           break;
 
         case MetricType::String:
-          metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.stringMetrics, deviceMetrics.stringTimestamps, mi, storeIdx);
+          fillNodeWithValue(w, filledMetrics, deviceMetrics.stringMetrics[storeIdx],
+                            deviceMetrics.stringTimestamps[storeIdx]);
           break;
 
         case MetricType::Uint64:
-          metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.uint64Metrics, deviceMetrics.uint64Timestamps, mi, storeIdx);
+          fillNodeWithValue(w, filledMetrics, deviceMetrics.uint64Metrics[storeIdx],
+                            deviceMetrics.uint64Timestamps[storeIdx]);
           break;
 
         default:
           continue;
       }
-      deviceRoot.add_child(std::string(metricLabel), metricNode);
     }
 
-    root.add_child(specs[idx].id, deviceRoot);
+    w.EndObject();
   }
 
-  boost::property_tree::ptree driverRoot;
+  w.Key("driver");
+  w.StartObject();
   for (size_t mi = 0; mi < driverMetrics.metricLabels.size(); mi++) {
     std::string_view const metricLabel{driverMetrics.metricLabels[mi].label, driverMetrics.metricLabels[mi].size};
     auto same = [metricLabel](std::regex const& matcher) -> bool {
@@ -130,39 +131,39 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric
 
     auto storeIdx = driverMetrics.metrics[mi].storeIdx;
     // and if data is there
-    if (driverMetrics.metrics[mi].filledMetrics == 0) {
+    size_t filledMetrics = driverMetrics.metrics[mi].filledMetrics;
+    if (filledMetrics == 0) {
       continue;
     }
 
-    // if so
-    boost::property_tree::ptree metricNode;
-
+    w.Key(metricLabel.data(), metricLabel.size());
     switch (driverMetrics.metrics[mi].type) {
       case MetricType::Int:
-        metricNode = fillNodeWithValue(driverMetrics, driverMetrics.intMetrics, driverMetrics.intTimestamps, mi, storeIdx);
+        fillNodeWithValue(w, filledMetrics, driverMetrics.intMetrics[storeIdx],
+                          driverMetrics.intTimestamps[storeIdx]);
         break;
 
       case MetricType::Float:
-        metricNode = fillNodeWithValue(driverMetrics, driverMetrics.floatMetrics, driverMetrics.floatTimestamps, mi, storeIdx);
+        fillNodeWithValue(w, filledMetrics, driverMetrics.floatMetrics[storeIdx],
+                          driverMetrics.floatTimestamps[storeIdx]);
         break;
 
       case MetricType::String:
-        metricNode = fillNodeWithValue(driverMetrics, driverMetrics.stringMetrics, driverMetrics.stringTimestamps, mi, storeIdx);
+        fillNodeWithValue(w, filledMetrics, driverMetrics.stringMetrics[storeIdx],
+                          driverMetrics.stringTimestamps[storeIdx]);
         break;
 
       case MetricType::Uint64:
-        metricNode = fillNodeWithValue(driverMetrics, driverMetrics.uint64Metrics, driverMetrics.uint64Timestamps, mi, storeIdx);
+        fillNodeWithValue(w, filledMetrics, driverMetrics.uint64Metrics[storeIdx],
+                          driverMetrics.uint64Timestamps[storeIdx]);
         break;
 
       default:
         continue;
     }
-    driverRoot.add_child(std::string{metricLabel}, metricNode);
   }
-
-  root.add_child("driver", driverRoot);
-
-  boost::property_tree::json_parser::write_json(out, root);
+  w.EndObject();
+  w.EndObject();
 
   return true;
 }

Framework/Core/test/test_ResourcesMonitoringHelpers.cxx

@@ -166,7 +166,8 @@ TEST_CASE("StreamMetrics")
   ResourcesMonitoringHelper::dumpMetricsToJSON(metrics,
                                                driverMetrics, specs, performanceMetrics,
                                                streamer);
-  REQUIRE(streamer.str() == R"JSON({
+  std::string streamed = streamer.str();
+  std::string expected = R"JSON({
     "someDevice": {
         "ckey": [
             {
@@ -266,5 +267,6 @@ TEST_CASE("StreamMetrics")
         ]
     }
 }
-)JSON");
+)JSON";
+  REQUIRE(std::regex_replace(streamed, std::regex(R"(\s+)"), "") == std::regex_replace(expected, std::regex(R"(\s+)"), ""));
 }