Benchmark Generators #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Generators | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| actions: write | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '25' | |
| - uses: jbangdev/setup-jbang@main | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libcairo2-dev | |
| - name: Install Python dependencies | |
| run: pip install pyyaml cairosvg | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| JAR="html-generators/generate.jar" | |
| AOT="html-generators/generate.aot" | |
| OG_JAR="html-generators/generateog.jar" | |
| OG_AOT="html-generators/generateog.aot" | |
| STEADY_RUNS=5 | |
| snippet_count=$(find content \( -name '*.json' -o -name '*.yaml' -o -name '*.yml' \) -not -name 'template.*' | wc -l | tr -d ' ') | |
| java_ver=$(java -version 2>&1 | head -1 | sed 's/.*"\(.*\)".*/\1/') | |
| os_name="ubuntu-latest" | |
| # Nanosecond timestamp helper | |
| _now() { python3 -c "import time; print(int(time.time()*1e9))"; } | |
| # Timing helper — returns seconds or "FAIL" | |
| measure() { | |
| local start end | |
| start=$(_now) | |
| if "$@" > /dev/null 2>&1; then | |
| end=$(_now) | |
| awk "BEGIN {printf \"%.2f\", ($end - $start) / 1000000000}" | |
| else | |
| echo "FAIL" | |
| fi | |
| } | |
| avg_runs() { | |
| local n="$1"; shift | |
| local sum=0 success=0 | |
| for ((i = 1; i <= n; i++)); do | |
| local t | |
| t=$(measure "$@") | |
| if [[ "$t" != "FAIL" ]]; then | |
| sum=$(awk "BEGIN {print $sum + $t}") | |
| success=$((success + 1)) | |
| fi | |
| done | |
| if [[ $success -eq 0 ]]; then | |
| echo "FAIL" | |
| else | |
| awk "BEGIN {printf \"%.2f\", $sum / $success}" | |
| fi | |
| } | |
| echo "Running benchmark on $os_name (Java $java_ver, $snippet_count snippets)..." | |
| # --- Phase 1: Training / build cost --- | |
| rm -f "$JAR" "$AOT" "$OG_JAR" "$OG_AOT" | |
| find html-generators -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true | |
| # HTML generator | |
| PY_TRAIN=$(measure python3 html-generators/generate.py) | |
| JBANG_EXPORT=$(measure jbang export fatjar --force --output "$JAR" html-generators/generate.java) | |
| AOT_TRAIN=$(measure java -XX:AOTCacheOutput="$AOT" -jar "$JAR") | |
| # OG generator | |
| OG_PY_TRAIN=$(measure python3 html-generators/generateog.py) | |
| OG_JBANG_EXPORT=$(measure jbang export fatjar --force --output "$OG_JAR" html-generators/generateog.java) | |
| OG_AOT_TRAIN=$(measure java -XX:AOTCacheOutput="$OG_AOT" -jar "$OG_JAR") | |
| # --- Phase 2: Steady-state execution --- | |
| # HTML generator | |
| PY_STEADY=$(avg_runs $STEADY_RUNS python3 html-generators/generate.py) | |
| JBANG_STEADY=$(avg_runs $STEADY_RUNS jbang html-generators/generate.java) | |
| JAR_STEADY=$(avg_runs $STEADY_RUNS java -XX:Tier4CompileThreshold=100 -jar "$JAR") | |
| AOT_STEADY=$(avg_runs $STEADY_RUNS java -XX:Tier4CompileThreshold=100 -XX:AOTCache="$AOT" -jar "$JAR") | |
| # OG generator | |
| OG_PY_STEADY=$(avg_runs $STEADY_RUNS python3 html-generators/generateog.py) | |
| OG_JBANG_STEADY=$(avg_runs $STEADY_RUNS jbang html-generators/generateog.java) | |
| OG_JAR_STEADY=$(avg_runs $STEADY_RUNS java -XX:Tier4CompileThreshold=100 -jar "$OG_JAR") | |
| OG_AOT_STEADY=$(avg_runs $STEADY_RUNS java -XX:Tier4CompileThreshold=100 -XX:AOTCache="$OG_AOT" -jar "$OG_JAR") | |
| # Write to GitHub Actions Job Summary | |
| { | |
| echo "## Benchmark Results — \`$os_name\`" | |
| echo "" | |
| echo "Java $java_ver · $snippet_count snippets" | |
| echo "" | |
| echo "### HTML Generator" | |
| echo "" | |
| echo "#### Phase 1: Training / Build Cost (one-time)" | |
| echo "" | |
| echo "| Step | Time | What it does |" | |
| echo "|------|------|-------------|" | |
| echo "| Python first run | ${PY_TRAIN}s | Interprets source, creates \`__pycache__\` bytecode |" | |
| echo "| JBang export | ${JBANG_EXPORT}s | Compiles source + bundles dependencies into fat JAR |" | |
| echo "| AOT training run | ${AOT_TRAIN}s | Runs JAR once to record class loading, produces \`.aot\` cache |" | |
| echo "" | |
| echo "#### Phase 2: Steady-State Execution (avg of $STEADY_RUNS runs)" | |
| echo "" | |
| echo "| Method | Avg Time |" | |
| echo "|--------|---------|" | |
| echo "| **Fat JAR + AOT** | **${AOT_STEADY}s** |" | |
| echo "| **Fat JAR** | ${JAR_STEADY}s |" | |
| echo "| **JBang** | ${JBANG_STEADY}s |" | |
| echo "| **Python** | ${PY_STEADY}s |" | |
| echo "" | |
| echo "### OG Card Generator" | |
| echo "" | |
| echo "#### Phase 1: Training / Build Cost (one-time)" | |
| echo "" | |
| echo "| Step | Time | What it does |" | |
| echo "|------|------|-------------|" | |
| echo "| Python first run | ${OG_PY_TRAIN}s | Interprets source, generates SVG+PNG via cairosvg |" | |
| echo "| JBang export | ${OG_JBANG_EXPORT}s | Compiles source + bundles Batik dependencies into fat JAR |" | |
| echo "| AOT training run | ${OG_AOT_TRAIN}s | Runs JAR once to record class loading, produces \`.aot\` cache |" | |
| echo "" | |
| echo "#### Phase 2: Steady-State Execution (avg of $STEADY_RUNS runs)" | |
| echo "" | |
| echo "| Method | Avg Time |" | |
| echo "|--------|---------|" | |
| echo "| **Fat JAR + AOT** | **${OG_AOT_STEADY}s** |" | |
| echo "| **Fat JAR** | ${OG_JAR_STEADY}s |" | |
| echo "| **JBang** | ${OG_JBANG_STEADY}s |" | |
| echo "| **Python** | ${OG_PY_STEADY}s |" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # --------------------------------------------------------------------------- | |
| # Phase 3: CI cold start — runs on a completely fresh runner. | |
| # JAR and AOT are built once then passed via artifact, simulating | |
| # the actions cache restore that happens in the deploy workflow. | |
| # Python and JBang start with zero caches, just like real CI. | |
| # --------------------------------------------------------------------------- | |
| build-jar: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '25' | |
| - uses: jbangdev/setup-jbang@main | |
| - name: Build fat JARs and AOT caches | |
| run: | | |
| jbang export fatjar --force --output html-generators/generate.jar html-generators/generate.java | |
| java -XX:AOTCacheOutput=html-generators/generate.aot -jar html-generators/generate.jar | |
| jbang export fatjar --force --output html-generators/generateog.jar html-generators/generateog.java | |
| java -XX:AOTCacheOutput=html-generators/generateog.aot -jar html-generators/generateog.jar | |
| - name: Upload JAR and AOT | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: generator | |
| path: | | |
| html-generators/generate.jar | |
| html-generators/generate.aot | |
| html-generators/generateog.jar | |
| html-generators/generateog.aot | |
| ci-cold-start: | |
| needs: build-jar | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '25' | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libcairo2-dev | |
| - name: Install Python dependencies | |
| run: pip install pyyaml cairosvg | |
| - name: Download JAR and AOT | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: generator | |
| path: html-generators | |
| - name: CI cold-start benchmark | |
| shell: bash | |
| run: | | |
| os_name="ubuntu-latest" | |
| java_ver=$(java -version 2>&1 | head -1 | sed 's/.*"\(.*\)".*/\1/') | |
| snippet_count=$(find content \( -name '*.json' -o -name '*.yaml' -o -name '*.yml' \) -not -name 'template.*' | wc -l | tr -d ' ') | |
| _now() { python3 -c "import time; print(int(time.time()*1e9))"; } | |
| measure() { | |
| local start end | |
| start=$(_now) | |
| if "$@" > /dev/null 2>&1; then | |
| end=$(_now) | |
| awk "BEGIN {printf \"%.2f\", ($end - $start) / 1000000000}" | |
| else | |
| echo "FAIL" | |
| fi | |
| } | |
| # Everything is cold: no __pycache__, no JBang cache, fresh JVM | |
| # HTML generator | |
| PY_CI=$(measure python3 html-generators/generate.py) | |
| JAR_CI=$(measure java -XX:Tier4CompileThreshold=100 -jar html-generators/generate.jar) | |
| AOT_CI=$(measure java -XX:Tier4CompileThreshold=100 -XX:AOTCache=html-generators/generate.aot -jar html-generators/generate.jar) | |
| # OG generator | |
| OG_PY_CI=$(measure python3 html-generators/generateog.py) | |
| OG_JAR_CI=$(measure java -XX:Tier4CompileThreshold=100 -jar html-generators/generateog.jar) | |
| OG_AOT_CI=$(measure java -XX:Tier4CompileThreshold=100 -XX:AOTCache=html-generators/generateog.aot -jar html-generators/generateog.jar) | |
| { | |
| echo "## CI Cold Start — \`$os_name\`" | |
| echo "" | |
| echo "Java $java_ver · $snippet_count snippets" | |
| echo "" | |
| echo "Fresh runner, no caches. JARs and AOT restored from artifact" | |
| echo "(simulates actions cache restore in deploy workflow)." | |
| echo "" | |
| echo "### HTML Generator" | |
| echo "" | |
| echo "| Method | Time |" | |
| echo "|--------|------|" | |
| echo "| **Fat JAR + AOT** | **${AOT_CI}s** |" | |
| echo "| **Fat JAR** | ${JAR_CI}s |" | |
| echo "| **Python** | ${PY_CI}s |" | |
| echo "" | |
| echo "### OG Card Generator" | |
| echo "" | |
| echo "| Method | Time |" | |
| echo "|--------|------|" | |
| echo "| **Fat JAR + AOT** | **${OG_AOT_CI}s** |" | |
| echo "| **Fat JAR** | ${OG_JAR_CI}s |" | |
| echo "| **Python** | ${OG_PY_CI}s |" | |
| } >> "$GITHUB_STEP_SUMMARY" |