chore: upgrade the benchmark

.github/workflows/performance.yaml

@@ -8,6 +8,10 @@ on:
   pull_request:
     branches:
       - main
+    # `labeled` lets applying the `performance-benchmark` label trigger a rerun
+    # without needing an empty commit; the job-level `if` filters out unrelated
+    # label changes so we don't burn CI on every label add.
+    types: [opened, synchronize, reopened, labeled]
 
 env:
   CI: true
@@ -16,6 +20,9 @@ env:
 jobs:
   historical-versions:
     runs-on: ubuntu-latest
+    # Skip the job when a `labeled` event fires for an unrelated label —
+    # opens/pushes/reopens always run; only the label firehose is filtered.
+    if: github.event.action != 'labeled' || github.event.label.name == 'performance-benchmark'
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Install pnpm for Rebilly/api-definitions
@@ -33,8 +40,9 @@ jobs:
         run: npm i -g hyperfine
 
       - name: Add more versions to test
-        # Run only on the release branch (changeset-release/main):
-        if: github.head_ref == 'changeset-release/main'
+        # Runs on the release PR (changeset-release/main) automatically, or on
+        # any PR carrying the `performance-benchmark` label for opt-in deep-dive.
+        if: github.head_ref == 'changeset-release/main' || contains(github.event.pull_request.labels.*.name, 'performance-benchmark')
         run: |
           cd tests/performance/
           cat package.json | jq ".dependencies = $(cat package.json | jq ._enhancedDependencies)" > package.json
@@ -56,8 +64,12 @@ jobs:
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 
       - name: Comment PR
+        # If the PR carries the `performance-benchmark` label, each push posts a
+        # fresh comment (so variance across reruns can be compared in-thread);
+        # otherwise the comment is overwritten in place so normal PRs keep a
+        # single result.
         if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1
         with:
           file-path: tests/performance/benchmark_chart.md
-          comment-tag: historical-versions-comparison
+          comment-tag: ${{ contains(github.event.pull_request.labels.*.name, 'performance-benchmark') && format('historical-versions-rerun-{0}', github.run_id) || 'historical-versions-comparison' }}

tests/performance/chart.js

@@ -2,16 +2,28 @@ import fs from 'node:fs';
 
 const content = fs.readFileSync('benchmark_check.json', 'utf8');
 const json = JSON.parse(content);
+
+/** Returns the median of a numeric array without mutating the input. */
+const median = (xs) => {
+  const sorted = [...xs].sort((a, b) => a - b);
+  const mid = sorted.length >> 1;
+  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+
+/** Median absolute deviation around a given centre — a robust alternative to stddev. */
+const medianAbsoluteDeviation = (xs, centre) => median(xs.map((x) => Math.abs(x - centre)));
+
 const arr = json.results.map((r) => [
   r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, cliVersion) => cliVersion),
-  r.mean,
-  r.stddev,
+  r.median,
+  medianAbsoluteDeviation(r.times, r.median),
 ]);
-const minMean = Math.min(...arr.map(([_, mean]) => mean));
+const minMedian = Math.min(...arr.map(([_, m]) => m));
 
-const constructBarForChart = (mean, min) => {
+/** Builds a unicode bar whose length scales with how much slower a result is than the fastest. */
+const constructBarForChart = (value, min) => {
   if (min <= 0) return 'N/A';
-  const slownessRatio = mean / min;
+  const slownessRatio = value / min;
   const slownessFactor = slownessRatio - 1;
   const maxBarLength = 30;
   const visualFactor = Math.min(1, slownessFactor);
@@ -20,19 +32,19 @@ const constructBarForChart = (mean, min) => {
 };
 
 const output = [
-  '| CLI Version | Mean Time ± Std Dev (s) | Relative Performance (Lower is Faster) |',
+  '| CLI Version | Median Time ± MAD (s) | Relative Performance (Lower is Faster) |',
   '|---|---|---|',
-  ...arr.map(([cliVersion, mean, stddev]) => {
-    const bar = constructBarForChart(mean, minMean);
-    const meanFormatted = mean.toFixed(3);
-    const stddevFormatted = stddev.toFixed(3);
-    const relativeSpeedFactor = (mean / minMean).toFixed(2);
-    const factorSuffix = mean === minMean ? 'x (Fastest)' : 'x';
-
-    const timeWithStddev = `${meanFormatted}s ± ${stddevFormatted}s`;
+  ...arr.map(([cliVersion, medianValue, madValue]) => {
+    const bar = constructBarForChart(medianValue, minMedian);
+    const medianFormatted = medianValue.toFixed(3);
+    const madFormatted = madValue.toFixed(3);
+    const relativeSpeedFactor = (medianValue / minMedian).toFixed(2);
+    const factorSuffix = medianValue === minMedian ? 'x (Fastest)' : 'x';
+
+    const timeWithSpread = `${medianFormatted}s ± ${madFormatted}s`;
     const performanceDisplay = `${bar} ${relativeSpeedFactor}${factorSuffix}`;
 
-    return `| ${cliVersion} | ${timeWithStddev} | ${performanceDisplay} |`;
+    return `| ${cliVersion} | ${timeWithSpread} | ${performanceDisplay} |`;
   }),
 ].join('\n');
 

tests/performance/make-test-command.sh

@@ -6,8 +6,10 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
-# Store the command into a text file:
-echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+# Store the command into a text file.
+# Pre-warm the OS page cache so the first measured run does not pay a cold-cache
+# penalty; with that done, --warmup 1 is enough to stabilise per-process state.
+echo "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null &&" REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json

tests/performance/package.json

@@ -7,9 +7,9 @@
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
-    "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
-    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
-    "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
+    "test:bundle": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
+    "test:lint": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
+    "test:check-config": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
   },
   "_enhancedDependencies": {
     "cli-2.0.0": "npm:@redocly/cli@2.0.0",