Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/frameworks.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"package": "app-astro",
"buildScript": "build",
"buildOutputDir": "dist",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand Down Expand Up @@ -74,7 +74,7 @@
"package": "app-next-js",
"buildScript": "build",
"buildOutputDir": ".next",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand All @@ -96,7 +96,7 @@
"package": "app-nuxt",
"buildScript": "build",
"buildOutputDir": ".output",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand All @@ -118,7 +118,7 @@
"package": "app-react-router",
"buildScript": "build",
"buildOutputDir": "build",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand All @@ -140,7 +140,7 @@
"package": "app-solid-start",
"buildScript": "build",
"buildOutputDir": ".output",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand All @@ -162,7 +162,7 @@
"package": "app-sveltekit",
"buildScript": "build",
"buildOutputDir": "build",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
},
{
Expand All @@ -184,7 +184,7 @@
"package": "app-tanstack-start-react",
"buildScript": "build",
"buildOutputDir": ".output",
"measurements": [{ "type": "ssr" }, { "type": "spa" }]
"measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }]
}
}
]
3 changes: 3 additions & 0 deletions .github/workflows/generate-stats.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
ssr-matrix: ${{ steps.set-matrix.outputs.ssr }}
deps-matrix: ${{ steps.set-matrix.outputs.deps }}
spa-matrix: ${{ steps.set-matrix.outputs.spa }}
mpa-matrix: ${{ steps.set-matrix.outputs.mpa }}
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
Expand All @@ -40,6 +41,7 @@ jobs:
echo "ssr=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssr"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT
echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT
echo "spa=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["spa"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT
echo "mpa=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["mpa"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT

measure:
needs: setup
Expand All @@ -50,6 +52,7 @@ jobs:
ssr-matrix: ${{ needs.setup.outputs.ssr-matrix }}
deps-matrix: ${{ needs.setup.outputs.deps-matrix }}
spa-matrix: ${{ needs.setup.outputs.spa-matrix }}
mpa-matrix: ${{ needs.setup.outputs.mpa-matrix }}

generate-stats:
needs: [setup, measure]
Expand Down
60 changes: 57 additions & 3 deletions .github/workflows/measure-framework.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ on:
type: string
required: false
default: '[]'
mpa-matrix:
description: 'JSON array of frameworks to measure MPA paint and interaction performance'
type: string
required: false
default: '[]'

jobs:
measure-install:
Expand Down Expand Up @@ -205,13 +210,13 @@ jobs:
framework: ${{ fromJson(inputs.spa-matrix) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup pnpm
uses: pnpm/action-setup@v4
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0

- name: Setup Node.js
uses: actions/setup-node@v4
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: '24'
cache: 'pnpm'
Expand Down Expand Up @@ -243,3 +248,52 @@ jobs:
path: packages/${{ matrix.framework.package }}/ci-stats.json
retention-days: 1
if-no-files-found: error

measure-mpa:
if: inputs.mpa-matrix != '[]'
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/playwright:v1.49.0-noble
options: --ipc=host
strategy:
fail-fast: false
matrix:
framework: ${{ fromJson(inputs.mpa-matrix) }}
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup pnpm
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0

- name: Setup Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: '24'
cache: 'pnpm'

- name: Install workspace dependencies
run: pnpm install --frozen-lockfile

- name: Install package dependencies
working-directory: ./packages/${{ matrix.framework.package }}
run: pnpm install --frozen-lockfile --ignore-workspace

- name: Build app
working-directory: ./packages/${{ matrix.framework.package }}
run: pnpm build

- name: Run MPA benchmark
run: pnpm --filter @framework-tracker/stats-generator run:mpa ${{ matrix.framework.package }}
env:
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
RUNNER_LABEL: ubuntu-latest

- name: Upload MPA stats
uses: actions/upload-artifact@v4
with:
name: mpa-stats-${{ matrix.framework.name }}
path: packages/${{ matrix.framework.package }}/ci-stats.json
retention-days: 1
if-no-files-found: error
22 changes: 22 additions & 0 deletions .github/workflows/validate-stats.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,28 @@ jobs:
(cd packages/$PKG && npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true)
done

echo ""
echo "=== Running SPA benchmarks ==="
FRAMEWORKS=$(cat .github/frameworks.json)
echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["spa"]))' | while read -r framework; do
PKG=$(echo "$framework" | jq -r '.starter.package')
echo "Building SPA for $PKG..."
(cd packages/$PKG && pnpm build)
echo "Running SPA benchmark for $PKG..."
pnpm --filter @framework-tracker/stats-generator run:spa $PKG
done

echo ""
echo "=== Running MPA benchmarks ==="
FRAMEWORKS=$(cat .github/frameworks.json)
echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["mpa"]))' | while read -r framework; do
PKG=$(echo "$framework" | jq -r '.app.package')
echo "Building MPA for $PKG..."
(cd packages/$PKG && pnpm build)
echo "Running MPA benchmark for $PKG..."
pnpm --filter @framework-tracker/stats-generator run:mpa $PKG
done

echo ""
echo "=== Validating all outputs ==="
pnpm --filter @framework-tracker/stats-generator validate
10 changes: 7 additions & 3 deletions packages/app-astro/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
{
"timingMeasuredAt": "2026-03-08T00:31:57.171Z",
"runner": "ubuntu-latest",
"timingMeasuredAt": "2026-04-21T05:06:35.326Z",
"runner": "local",
"frameworkVersion": "5.16.15",
"ssrOpsPerSec": 366,
"ssrAvgLatencyMs": 2.735,
"ssrSamples": 3656,
"ssrBodySizeKb": 99.86,
"ssrDuplicationFactor": 1
"ssrDuplicationFactor": 1,
"mpaFirstPaintMs": 90.33,
"mpaFCPMs": 90.39,
"mpaINPMs": 22.56,
"mpaRuns": 3
}
8 changes: 6 additions & 2 deletions packages/app-next-js/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"timingMeasuredAt": "2026-04-05T04:53:54.987Z",
"timingMeasuredAt": "2026-04-21T05:07:39.298Z",
"runner": "local",
"frameworkVersion": "16.1.1",
"ssrOpsPerSec": 129,
Expand All @@ -10,5 +10,9 @@
"spaFirstPaintMs": 371,
"spaFCPMs": 370.94,
"spaINPMs": 23.38,
"spaRuns": 1
"spaRuns": 1,
"mpaFirstPaintMs": 127.33,
"mpaFCPMs": 127.19,
"mpaINPMs": 20.21,
"mpaRuns": 3
}
14 changes: 11 additions & 3 deletions packages/app-nuxt/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
{
"timingMeasuredAt": "2026-03-08T00:31:57.171Z",
"runner": "ubuntu-latest",
"timingMeasuredAt": "2026-04-21T05:26:28.141Z",
"runner": "local",
"frameworkVersion": "4.2.2",
"ssrOpsPerSec": 248,
"ssrAvgLatencyMs": 4.037,
"ssrSamples": 2478,
"ssrBodySizeKb": 201.18,
"ssrDuplicationFactor": 2
"ssrDuplicationFactor": 2,
"mpaFirstPaintMs": 89.67,
"mpaFCPMs": 89.6,
"mpaINPMs": 24.05,
"mpaRuns": 3,
"spaFirstPaintMs": 111.67,
"spaFCPMs": 111.7,
"spaINPMs": 22.65,
"spaRuns": 3
}
14 changes: 11 additions & 3 deletions packages/app-react-router/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
{
"timingMeasuredAt": "2026-03-08T00:31:57.171Z",
"runner": "ubuntu-latest",
"timingMeasuredAt": "2026-04-21T05:29:46.068Z",
"runner": "local",
"frameworkVersion": "7.10.1",
"ssrOpsPerSec": 64,
"ssrAvgLatencyMs": 15.528,
"ssrSamples": 644,
"ssrBodySizeKb": 211.14,
"ssrDuplicationFactor": 2
"ssrDuplicationFactor": 2,
"mpaFirstPaintMs": 167.33,
"mpaFCPMs": 167.24,
"mpaINPMs": 24.37,
"mpaRuns": 3,
"spaFirstPaintMs": 121,
"spaFCPMs": 121.16,
"spaINPMs": 22.24,
"spaRuns": 3
}
14 changes: 11 additions & 3 deletions packages/app-solid-start/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
{
"timingMeasuredAt": "2026-03-08T00:31:57.171Z",
"runner": "ubuntu-latest",
"timingMeasuredAt": "2026-04-21T05:30:50.803Z",
"runner": "local",
"frameworkVersion": "1.2.1",
"ssrOpsPerSec": 234,
"ssrAvgLatencyMs": 4.275,
"ssrSamples": 2340,
"ssrBodySizeKb": 225.49,
"ssrDuplicationFactor": 2
"ssrDuplicationFactor": 2,
"mpaFirstPaintMs": 106.33,
"mpaFCPMs": 106.31,
"mpaINPMs": 23.79,
"mpaRuns": 3,
"spaFirstPaintMs": 114,
"spaFCPMs": 114.32,
"spaINPMs": 21.33,
"spaRuns": 3
}
8 changes: 6 additions & 2 deletions packages/app-sveltekit/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"timingMeasuredAt": "2026-04-05T04:55:06.141Z",
"timingMeasuredAt": "2026-04-21T05:11:57.395Z",
"runner": "local",
"frameworkVersion": "2.49.4",
"ssrOpsPerSec": 259,
Expand All @@ -10,5 +10,9 @@
"spaFirstPaintMs": 93,
"spaFCPMs": 93.14,
"spaINPMs": 20.37,
"spaRuns": 1
"spaRuns": 1,
"mpaFirstPaintMs": 109,
"mpaFCPMs": 108.78,
"mpaINPMs": 21.66,
"mpaRuns": 3
}
14 changes: 11 additions & 3 deletions packages/app-tanstack-start-react/ci-stats.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
{
"timingMeasuredAt": "2026-03-08T00:31:57.171Z",
"runner": "ubuntu-latest",
"timingMeasuredAt": "2026-04-21T05:31:57.912Z",
"runner": "local",
"frameworkVersion": "1.145.3",
"ssrOpsPerSec": 185,
"ssrAvgLatencyMs": 5.395,
"ssrSamples": 1854,
"ssrBodySizeKb": 193.53,
"ssrDuplicationFactor": 2
"ssrDuplicationFactor": 2,
"mpaFirstPaintMs": 109.33,
"mpaFCPMs": 109.36,
"mpaINPMs": 23.98,
"mpaRuns": 3,
"spaFirstPaintMs": 693,
"spaFCPMs": 693.07,
"spaINPMs": 104.6,
"spaRuns": 3
}
1 change: 1 addition & 0 deletions packages/stats-generator/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"collect": "node src/collect-stats.ts",
"run:ssr": "node src/run-ssr-benchmark.ts",
"run:spa": "node src/run-spa-benchmark.ts",
"run:mpa": "node src/run-mpa-benchmark.ts",
"run:install": "node src/run-install-benchmark.ts",
"run:build": "node src/run-build-benchmark.ts",
"run:corejs": "node src/run-corejs-scan.ts",
Expand Down
Loading
Loading