[SVLS-8351] Add CPU Enhanced Metrics in Linux Azure Functions

.github/workflows/build-datadog-serverless-compat.yml

@@ -45,7 +45,7 @@ jobs:
           retention-days: 3
       - if: ${{ inputs.runner == 'windows-2022' }}
         shell: bash
-        run: cargo build --release -p datadog-serverless-compat --features windows-pipes
+        run: cargo build --release -p datadog-serverless-compat --features windows-pipes,windows-enhanced-metrics
       - if: ${{ inputs.runner == 'windows-2022' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2
         with:

.github/workflows/cargo.yml

@@ -95,7 +95,7 @@ jobs:
       - shell: bash
         run: |
           if [[ "${{ inputs.runner }}" == "windows-2022" ]]; then
-            cargo nextest run --workspace --features datadog-serverless-compat/windows-pipes
+            cargo nextest run --workspace --features datadog-serverless-compat/windows-pipes,datadog-serverless-compat/windows-enhanced-metrics
           else
             cargo nextest run --workspace
           fi

LICENSE-3rdparty.csv

@@ -69,6 +69,7 @@ headers,https://github.com/hyperium/headers,MIT,Sean McArthur <sean@seanmonstar.
 headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur <sean@seanmonstar.com>
 heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors
 heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats <woboats@gmail.com>
+hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes
 hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi <kokakiwi@kokakiwi.net>
 home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson <andersrb@gmail.com>
 http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Carl Lerche <me@carllerche.com>, Sean McArthur <sean@seanmonstar.com>"
@@ -119,6 +120,7 @@ multimap,https://github.com/havarnov/multimap,MIT OR Apache-2.0,Håvar Nøvik <h
 nix,https://github.com/nix-rust/nix,MIT,The nix-rust Project Developers
 nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) <ryan.havvy@gmail.com>, Josh Triplett <josh@joshtriplett.org>, The Nushell Project Developers"
 num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers
+num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur <sean@seanmonstar.com>
 once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov <aleksey.kladov@gmail.com>
 openssl-probe,https://github.com/rustls/openssl-probe,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
 ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem <jonathan.reem@gmail.com>, Matt Brubeck <mbrubeck@limpet.net>"

crates/datadog-metrics-collector/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "datadog-metrics-collector"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+description = "Collector to read, compute, and submit enhanced metrics in Serverless environments"
+
+[dependencies]
+dogstatsd = { path = "../dogstatsd", default-features = true }
+num_cpus = "1.16"
+tracing = { version = "0.1", default-features = false }
+libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", default-features = false }
+
+[features]
+windows-enhanced-metrics = []

crates/datadog-metrics-collector/src/cpu.rs

@@ -0,0 +1,167 @@
+// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+//! CPU metrics collector for Azure Functions
+//!
+//! This module provides OS-agnostic CPU stats collection, CPU usage
+//! and limit computation, and metrics submission to Datadog.
+//!
+//! All CPU metrics are reported in nanocores (1 core = 1,000,000,000 nanocores).
+
+use dogstatsd::aggregator::AggregatorHandle;
+use dogstatsd::metric::{Metric, MetricValue, SortedTags};
+use libdd_common::azure_app_services;
+use std::env;
+use tracing::{debug, error};
+
+const CPU_USAGE_METRIC: &str = "azure.functions.enhanced.cpu.usage";
+const CPU_LIMIT_METRIC: &str = "azure.functions.enhanced.cpu.limit";
+
+/// Computed CPU total and limit metrics
+pub struct CpuStats {
+    pub total: u64,            // Cumulative CPU usage in nanoseconds
+    pub limit: Option<f64>,    // CPU limit in nanocores
+    pub defaulted_limit: bool, // Whether CPU limit was defaulted to host CPU count
+}
+
+pub trait CpuStatsReader {
+    fn read(&self) -> Option<CpuStats>;
+}
+
+pub struct CpuMetricsCollector {
+    reader: Box<dyn CpuStatsReader>,
+    aggregator: AggregatorHandle,
+    tags: Option<SortedTags>,
+    last_usage_ns: Option<u64>,
+    last_collection_time: std::time::Instant,
+}
+
+impl CpuMetricsCollector {
+    /// Creates a new CpuMetricsCollector
+    ///
+    /// # Arguments
+    ///
+    /// * `aggregator` - The aggregator handle to submit metrics to
+    /// * `tags` - Optional tags to attach to all metrics
+    pub fn new(aggregator: AggregatorHandle, tags: Option<SortedTags>) -> Self {
+        #[cfg(feature = "windows-enhanced-metrics")]
+        let reader: Box<dyn CpuStatsReader> = Box::new(crate::windows::WindowsCpuStatsReader);
+        #[cfg(not(feature = "windows-enhanced-metrics"))]
+        let reader: Box<dyn CpuStatsReader> = Box::new(crate::linux::LinuxCpuStatsReader);
+        Self {
+            reader,
+            aggregator,
+            tags,
+            last_usage_ns: None,
+            last_collection_time: std::time::Instant::now(),
+        }
+    }
+
+    pub fn collect_and_submit(&mut self) {
+        if let Some(cpu_stats) = self.reader.read() {
+            // Submit metrics
+            let current_usage_ns = cpu_stats.total;
+            let now_instant = std::time::Instant::now();
+
+            // Skip first collection
+            let Some(last_usage_ns) = self.last_usage_ns else {
+                debug!("First CPU collection, skipping interval");
+                self.last_usage_ns = Some(current_usage_ns);
+                self.last_collection_time = now_instant;
+                return;
+            };
+
+            if current_usage_ns < last_usage_ns {
+                debug!("Current CPU usage is less than last usage, skipping interval");
+                self.last_usage_ns = Some(current_usage_ns);
+                self.last_collection_time = now_instant;
+                return;
+            }
+            let delta_ns = (current_usage_ns - last_usage_ns) as f64;
+            self.last_usage_ns = Some(current_usage_ns);
+            let elapsed_secs = now_instant
+                .duration_since(self.last_collection_time)
+                .as_secs_f64();
+            self.last_collection_time = now_instant;
+
+            // Divide nanoseconds delta by elapsed time to get usage rate in nanocores
+            let usage_rate_nc = delta_ns / elapsed_secs;
+
+            let now = std::time::UNIX_EPOCH
+                .elapsed()
+                .map(|d| d.as_secs())
+                .unwrap_or(0)
+                .try_into()
+                .unwrap_or(0);
+
+            let usage_metric = Metric::new(
+                CPU_USAGE_METRIC.into(),
+                MetricValue::distribution(usage_rate_nc),
+                self.tags.clone(),
+                Some(now),
+            );
+
+            if let Err(e) = self.aggregator.insert_batch(vec![usage_metric]) {
+                error!("Failed to insert CPU usage metric: {}", e);
+            }
+
+            if let Some(limit) = cpu_stats.limit {
+                if cpu_stats.defaulted_limit {
+                    debug!("CPU limit defaulted to host CPU count");
+                }
+                let limit_metric = Metric::new(
+                    CPU_LIMIT_METRIC.into(),
+                    MetricValue::distribution(limit),
+                    self.tags.clone(),
+                    Some(now),
+                );
+                if let Err(e) = self.aggregator.insert_batch(vec![limit_metric]) {
+                    error!("Failed to insert CPU limit metric: {}", e);
+                }
+            }
+        } else {
+            debug!(
+                "Skipping CPU metrics collection - could not find data to generate CPU usage and limit enhanced metrics"
+            );
+        }
+    }
+}
+
+pub fn build_cpu_metrics_tags() -> Option<SortedTags> {
+    let mut tag_parts = Vec::new();
+    // Azure tags from ddcommon
+    if let Some(aas_metadata) = &*azure_app_services::AAS_METADATA_FUNCTION {
+        let aas_tags = [
+            ("resource_id", aas_metadata.get_resource_id()),
+            ("resource_group", aas_metadata.get_resource_group()),
+            ("subscription_id", aas_metadata.get_subscription_id()),
+            ("name", aas_metadata.get_site_name()),
+        ];
+        for (name, value) in aas_tags {
+            if value != "unknown" {
+                tag_parts.push(format!("{}:{}", name, value));
+            }
+        }
+    }
+
+    // Tags from env vars (not in ddcommon) - origin tag is added by DogStatsD
+    for (tag_name, env_var) in [
+        ("region", "REGION_NAME"),
+        ("plan_tier", "WEBSITE_SKU"),
+        ("service", "DD_SERVICE"),
+        ("env", "DD_ENV"),
+        ("version", "DD_VERSION"),
+        ("serverless_compat_version", "DD_SERVERLESS_COMPAT_VERSION"),
+    ] {
+        if let Ok(val) = env::var(env_var) {
+            if !val.is_empty() {
+                tag_parts.push(format!("{}:{}", tag_name, val));
+            }
+        }
+    }
+
+    if tag_parts.is_empty() {
+        return None;
+    }
+    SortedTags::parse(&tag_parts.join(",")).ok()
+}

crates/datadog-metrics-collector/src/lib.rs

@@ -0,0 +1,14 @@
+// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+#![cfg_attr(not(test), deny(clippy::panic))]
+#![cfg_attr(not(test), deny(clippy::unwrap_used))]
+#![cfg_attr(not(test), deny(clippy::expect_used))]
+#![cfg_attr(not(test), deny(clippy::todo))]
+#![cfg_attr(not(test), deny(clippy::unimplemented))]
+
+pub mod cpu;
+#[cfg(not(feature = "windows-enhanced-metrics"))]
+pub(crate) mod linux;
+#[cfg(feature = "windows-enhanced-metrics")]
+pub(crate) mod windows;