Grafana: pending-priority stat for Overview Job Queue panel

[lukemelia]

Summary

• Replace the Job Queue timeseries on the Overview dashboard with a stat panel whose big number is pending priority jobs (status='unfulfilled', no active reservation, priority < 0)
• Threshold-driven color: green ≤3, yellow >3, red >10
• Sparkline keeps the resolved jobs/min activity shape, with the trailing point pinned to the current pending-priority count so lastNotNull matches the right edge of the line

The full Jobs/min + Failures/min timeseries view is still available on the Job Queue deep-dive dashboard linked from this panel.

Implementation note

$__timeGroupAlias for Postgres emits numeric epoch seconds, not a timestamp, so the UNION's appended row uses EXTRACT(EPOCH FROM NOW()) — using NOW()::timestamp produces UNION types numeric and timestamp without time zone cannot be matched (SQLSTATE 42804).

Test plan

• Open the Overview dashboard locally — Job Queue tile shows the pending-priority count
• Color reflects threshold: green when count ≤3, yellow when 4–10, red when ≥11 (verify by inserting test jobs or checking against current state)
• Sparkline tracks resolved jobs/min activity over the dashboard time range
• Click-through still navigates to the Job Queue deep-dive dashboard

🤖 Generated with Claude Code

[github-actions]

Observability diff (vs staging)

diff --git a/tmp/remote-canon.qttKuh/dashboards/boxel-status/overview.json b/tmp/committed-canon.sNiyy4/dashboards/boxel-status/overview.json
index b9556af..18ed932 100644
--- a/tmp/remote-canon.qttKuh/dashboards/boxel-status/overview.json
+++ b/tmp/committed-canon.sNiyy4/dashboards/boxel-status/overview.json
@@ -384,41 +384,13 @@
           "type": "grafana-postgresql-datasource",
           "uid": "cef5v5sl9k7i8f"
         },
-        "description": "Worker job throughput and failures, bucketed at 1 minute. Resolved jobs (`status='resolved'`) and rejected jobs (`status='rejected'`) stacked. Legend shows the mean over the visible time range. Counts ALL job types, not just indexing — for indexing-only throughput see the Indexing dashboard.",
+        "description": "Priority-pending worker jobs (status='unfulfilled' with no active reservation AND priority < 0). Sparkline shows resolved jobs/min over the dashboard time range, with the trailing point set to the current priority-pending count so `lastNotNull` matches the displayed value. Color thresholds: green ≤3, yellow >3, red >10. Counts ALL job types — for indexing-only depth see the Indexing dashboard.",
         "fieldConfig": {
           "defaults": {
             "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 50,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "smooth",
-              "lineWidth": 2,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "never",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
+              "mode": "thresholds"
             },
-            "decimals": 1,
+            "decimals": 0,
             "links": [
               {
                 "targetBlank": false,
@@ -432,43 +404,20 @@
               "steps": [
                 {
                   "color": "green"
+                },
+                {
+                  "color": "yellow",
+                  "value": 4
+                },
+                {
+                  "color": "red",
+                  "value": 11
                 }
               ]
             },
             "unit": "short"
           },
-          "overrides": [
-            {
-              "matcher": {
-                "id": "byName",
-                "options": "Failures/min"
-              },
-              "properties": [
-                {
-                  "id": "color",
-                  "value": {
-                    "fixedColor": "#ff5050",
-                    "mode": "fixed"
-                  }
-                }
-              ]
-            },
-            {
-              "matcher": {
-                "id": "byName",
-                "options": "Jobs/min"
-              },
-              "properties": [
-                {
-                  "id": "color",
-                  "value": {
-                    "fixedColor": "#37eb77",
-                    "mode": "fixed"
-                  }
-                }
-              ]
-            }
-          ]
+          "overrides": []
         },
         "gridPos": {
           "h": 8,
@@ -487,18 +436,19 @@
           }
         ],
         "options": {
-          "legend": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "vertical",
+          "reduceOptions": {
             "calcs": [
-              "mean"
+              "lastNotNull"
             ],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
+            "fields": "",
+            "values": false
           },
-          "tooltip": {
-            "mode": "multi",
-            "sort": "none"
-          }
+          "textMode": "value_and_name",
+          "wideLayout": true
         },
         "pluginVersion": "12.4.3",
         "targets": [
@@ -510,23 +460,12 @@
             "editorMode": "code",
             "format": "time_series",
             "rawQuery": true,
-            "rawSql": "SELECT $__timeGroupAlias(finished_at, '1m', 0),\n       COUNT(*) AS \"Jobs/min\"\n  FROM jobs\n WHERE status = 'resolved'\n   AND finished_at IS NOT NULL\n   AND $__timeFilter(finished_at)\n GROUP BY 1\n ORDER BY 1;",
+            "rawSql": "SELECT $__timeGroupAlias(finished_at, '1m', 0),\n       COUNT(*)::int AS \"Pending priority jobs\"\n  FROM jobs\n WHERE status = 'resolved'\n   AND finished_at IS NOT NULL\n   AND $__timeFilter(finished_at)\n GROUP BY 1\nUNION ALL\nSELECT EXTRACT(EPOCH FROM NOW()),\n       (SELECT COUNT(*)::int\n          FROM jobs j\n          LEFT JOIN job_reservations jr\n            ON j.id = jr.job_id\n           AND jr.completed_at IS NULL\n           AND jr.locked_until > NOW()\n         WHERE j.status = 'unfulfilled'\n           AND j.priority < 0\n           AND jr.id IS NULL)\n ORDER BY 1;",
             "refId": "A"
-          },
-          {
-            "datasource": {
-              "type": "grafana-postgresql-datasource",
-              "uid": "cef5v5sl9k7i8f"
-            },
-            "editorMode": "code",
-            "format": "time_series",
-            "rawQuery": true,
-            "rawSql": "SELECT $__timeGroupAlias(finished_at, '1m', 0),\n       COUNT(*) AS \"Failures/min\"\n  FROM jobs\n WHERE status = 'rejected'\n   AND finished_at IS NOT NULL\n   AND $__timeFilter(finished_at)\n GROUP BY 1\n ORDER BY 1;",
-            "refId": "B"
           }
         ],
         "title": "Job Queue",
-        "type": "timeseries"
+        "type": "stat"
       },
       {
         "datasource": {
@@ -1935,8 +1874,7 @@
                 }
               ]
             }
-          },
-          "overrides": []
+          }
         },
         "gridPos": {
           "h": 4,
@@ -2003,8 +1941,7 @@
                 }
               ]
             }
-          },
-          "overrides": []
+          }
         },
         "gridPos": {
           "h": 4,
@@ -2080,8 +2017,7 @@
               ]
             },
             "unit": "s"
-          },
-          "overrides": []
+          }
         },
         "gridPos": {
           "h": 4,
@@ -2152,8 +2088,7 @@
                 }
               ]
             }
-          },
-          "overrides": []
+          }
         },
         "gridPos": {
           "h": 4,
@@ -2388,7 +2323,7 @@
         {
           "hide": 2,
           "name": "env",
-          "query": "staging",
+          "query": "__ENV__",
           "skipUrlSync": true,
           "type": "constant"
         }

(Run: https://github.com/cardstack/boxel/actions/runs/25648422653)