feat: add check_http_metrics task for Prometheus metrics assertions

go.mod

@@ -4,6 +4,7 @@ go 1.25.7
 
 require (
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0
+	github.com/dustin/go-humanize v1.0.1
 	github.com/ethereum/go-ethereum v1.17.3-0.20260421080339-499762852cf2
 	github.com/ethpandaops/ethwallclock v0.4.0
 	github.com/ethpandaops/go-eth2-client v0.1.2
@@ -23,10 +24,13 @@ require (
 	github.com/mashingan/smapping v0.1.19
 	github.com/pressly/goose/v3 v3.27.1
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
+	github.com/prometheus/common v0.66.1
 	github.com/protolambda/zrnt v0.34.1
 	github.com/protolambda/ztyp v0.2.2
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15
 	github.com/rs/zerolog v1.35.1
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sirupsen/logrus v1.9.4
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
@@ -37,6 +41,7 @@ require (
 	github.com/wealdtech/go-eth2-types/v2 v2.8.2
 	github.com/wealdtech/go-eth2-util v1.8.2
 	golang.org/x/text v0.37.0
+	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -57,7 +62,6 @@ require (
 	github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/dot v1.6.4 // indirect
 	github.com/ethereum/c-kzg-4844/v2 v2.1.6 // indirect
 	github.com/ferranbt/fastssz v0.1.4 // indirect
@@ -89,13 +93,10 @@ require (
 	github.com/pk910/dynamic-ssz v1.3.2-0.20260505131440-111bcb265c8f // indirect
 	github.com/pk910/hashtree-bindings v0.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect
 	github.com/protolambda/bls12-381-util v0.1.0 // indirect
 	github.com/r3labs/sse/v2 v2.10.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/supranational/blst v0.3.16 // indirect
@@ -118,7 +119,6 @@ require (
 	golang.org/x/sys v0.43.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	modernc.org/libc v1.72.1 // indirect
 	modernc.org/mathutil v1.7.1 // indirect

pkg/tasks/check_http_metrics/README.md

@@ -0,0 +1,131 @@
+## `check_http_metrics` Task
+
+### Description
+The `check_http_metrics` task fetches metrics from an HTTP Prometheus endpoint and evaluates assertions against metric values.
+
+#### Task Behavior
+- The task polls the metrics endpoint at regular intervals.
+- By default, the task returns immediately when all assertions pass.
+- Use `continueOnPass: true` to keep monitoring even after success.
+- Use `failOnCheckMiss: true` to fail immediately when assertions are not met.
+
+### Configuration Parameters
+
+- **`url`**:\
+  HTTP URL of the Prometheus metrics endpoint. Required.
+
+- **`headers`**:\
+  Optional HTTP request headers (e.g., for authentication). Default: `{}`.
+
+- **`pollInterval`**:\
+  Interval between metric scrapes. Default: `10s`.
+
+- **`requestTimeout`**:\
+  Timeout for a single HTTP request. Default: `5s`.
+
+- **`maxResponseSize`**:\
+  Maximum response body size. Must be positive. Default: `10MB`.
+
+- **`failOnCheckMiss`**:\
+  If `true`, fail immediately when assertions are not met. If `false`, keep polling until timeout or success. Default: `false`.
+
+- **`continueOnPass`**:\
+  If `true`, continue checking after all assertions pass. Default: `false`.
+
+- **`missingMetric`**:\
+  Behavior when a metric family is missing: `wait`, `fail`, or `pass`. Default: `wait`.
+
+- **`missingSeries`**:\
+  Behavior when no time series matches the label selector: `wait`, `fail`, or `pass`. Default: `wait`.
+
+- **`resetBehavior`**:\
+  Behavior when a COUNTER metric's value drops below baseline (indicating restart): `fail`, `rebaseline`, or `ignore`. Only applies to COUNTER type metrics. Default: `fail`.
+
+- **`assertions`**:\
+  List of metric assertions. At least one required.
+
+#### Assertion Configuration
+
+- **`name`**: Unique assertion name. Required.
+- **`metric`**: Prometheus metric name. Required.
+- **`labels`**: Label selector (subset matching). Must match exactly one series.
+- **`mode`**: `value` (current value) or `delta` (change since baseline). Default: `value`.
+- **`operator`**: Comparison operator: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`. Required.
+- **`value`**: Expected numeric value. Required.
+- **`missingMetric`**: Per-assertion override for global `missingMetric`.
+- **`missingSeries`**: Per-assertion override for global `missingSeries`.
+
+#### Delta Mode
+
+In `delta` mode, the task tracks changes over time:
+1. First scrape: records the current value as baseline (waits, does not evaluate)
+2. Subsequent scrapes: computes `delta = current - baseline` and evaluates
+
+Negative deltas are valid for GAUGE and UNTYPED metrics. For COUNTER metrics, a decrease triggers `resetBehavior`.
+
+#### Examples
+
+```yaml
+assertions:
+  # Check counter increased by at least 1
+  - name: counter_increased
+    metric: my_counter
+    labels:
+      env: prod
+    mode: delta
+    operator: gte
+    value: 1
+
+  # Check gauge decreased (negative delta)
+  - name: gauge_dropped
+    metric: my_gauge
+    mode: delta
+    operator: lte
+    value: -1
+
+  # Check current value is above threshold
+  - name: value_above_threshold
+    metric: my_metric
+    operator: gt
+    value: 100
+```
+
+#### Metric Type Handling
+
+| Type | Value Extracted |
+|------|-----------------|
+| COUNTER | Counter value |
+| GAUGE | Gauge value |
+| UNTYPED | Untyped value |
+| SUMMARY | Sample sum |
+| HISTOGRAM | Sample sum |
+
+Counter reset detection only applies to COUNTER type. SUMMARY and HISTOGRAM use sample sum; bucket/quantile helpers are not supported.
+
+### Outputs
+
+- **`passedAssertions`**: Array of assertion names that passed.
+- **`failedAssertions`**: Array of assertion names that failed.
+- **`values`**: Map of assertion name to latest observed value.
+- **`deltas`**: Map of assertion name to computed delta (for `delta` mode).
+- **`baselines`**: Map of assertion name to baseline value (for `delta` mode).
+- **`scrapeErrors`**: Number of HTTP/parsing errors.
+- **`assertionErrors`**: Number of assertion evaluation errors.
+
+### Defaults
+
+```yaml
+- name: check_http_metrics
+  config:
+    url: ""
+    headers: {}
+    pollInterval: 10s
+    requestTimeout: 5s
+    maxResponseSize: 10MB
+    failOnCheckMiss: false
+    continueOnPass: false
+    missingMetric: wait
+    missingSeries: wait
+    resetBehavior: fail
+    assertions: []
+```