Benchmark Generators #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Generator | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| actions: write | |
| jobs: | |
| benchmark: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '25' | |
| - uses: jbangdev/setup-jbang@main | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| JAR="html-generators/generate.jar" | |
| AOT="html-generators/generate.aot" | |
| STEADY_RUNS=5 | |
| snippet_count=$(find content -name '*.json' | wc -l | tr -d ' ') | |
| java_ver=$(java -version 2>&1 | head -1 | sed 's/.*"\(.*\)".*/\1/') | |
| os_name="${{ matrix.os }}" | |
| # Timing helper using bash TIMEFORMAT | |
| measure() { | |
| TIMEFORMAT='%R' | |
| local t | |
| t=$( { time "$@" > /dev/null 2>&1; } 2>&1 ) | |
| echo "$t" | |
| } | |
| avg_runs() { | |
| local n="$1"; shift | |
| local sum=0 | |
| for ((i = 1; i <= n; i++)); do | |
| local t | |
| t=$(measure "$@") | |
| sum=$(awk "BEGIN {print $sum + $t}") | |
| done | |
| awk "BEGIN {printf \"%.2f\", $sum / $n}" | |
| } | |
| echo "Running benchmark on $os_name (Java $java_ver, $snippet_count snippets)..." | |
| # --- Phase 1: Training / build cost --- | |
| rm -f "$JAR" "$AOT" | |
| find html-generators -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true | |
| PY_TRAIN=$(measure python3 html-generators/generate.py) | |
| JBANG_EXPORT=$(measure jbang export fatjar --force --output "$JAR" html-generators/generate.java) | |
| AOT_TRAIN=$(measure java -XX:AOTCacheOutput="$AOT" -jar "$JAR") | |
| # --- Phase 2: Steady-state execution --- | |
| PY_STEADY=$(avg_runs $STEADY_RUNS python3 html-generators/generate.py) | |
| JBANG_STEADY=$(avg_runs $STEADY_RUNS jbang html-generators/generate.java) | |
| JAR_STEADY=$(avg_runs $STEADY_RUNS java -jar "$JAR") | |
| AOT_STEADY=$(avg_runs $STEADY_RUNS java -XX:AOTCache="$AOT" -jar "$JAR") | |
| # --- Phase 3: CI cold start (no caches, simulates fresh runner) --- | |
| find html-generators -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true | |
| PY_CI=$(measure python3 html-generators/generate.py) | |
| jbang cache clear > /dev/null 2>&1 || true | |
| JBANG_CI=$(measure jbang html-generators/generate.java) | |
| JAR_CI=$(measure java -jar "$JAR") | |
| AOT_CI=$(measure java -XX:AOTCache="$AOT" -jar "$JAR") | |
| # Write to GitHub Actions Job Summary | |
| { | |
| echo "## Benchmark Results — \`$os_name\`" | |
| echo "" | |
| echo "Java $java_ver · $snippet_count snippets" | |
| echo "" | |
| echo "### Phase 1: Training / Build Cost (one-time)" | |
| echo "" | |
| echo "| Step | Time | What it does |" | |
| echo "|------|------|-------------|" | |
| echo "| Python first run | ${PY_TRAIN}s | Interprets source, creates \`__pycache__\` bytecode |" | |
| echo "| JBang export | ${JBANG_EXPORT}s | Compiles source + bundles dependencies into fat JAR |" | |
| echo "| AOT training run | ${AOT_TRAIN}s | Runs JAR once to record class loading, produces \`.aot\` cache |" | |
| echo "" | |
| echo "### Phase 2: Steady-State Execution (avg of $STEADY_RUNS runs)" | |
| echo "" | |
| echo "| Method | Avg Time |" | |
| echo "|--------|---------|" | |
| echo "| **Fat JAR + AOT** | **${AOT_STEADY}s** |" | |
| echo "| **Fat JAR** | ${JAR_STEADY}s |" | |
| echo "| **JBang** | ${JBANG_STEADY}s |" | |
| echo "| **Python** | ${PY_STEADY}s |" | |
| echo "" | |
| echo "### Phase 3: CI Cold Start (fresh runner, no caches)" | |
| echo "" | |
| echo "Simulates a CI environment where every run is the first run." | |
| echo "Python has no \`__pycache__\`, JBang has no compilation cache." | |
| echo "Java AOT benefits from the pre-built \`.aot\` file restored from actions cache." | |
| echo "" | |
| echo "| Method | Time |" | |
| echo "|--------|------|" | |
| echo "| **Fat JAR + AOT** | **${AOT_CI}s** |" | |
| echo "| **Fat JAR** | ${JAR_CI}s |" | |
| echo "| **JBang** | ${JBANG_CI}s |" | |
| echo "| **Python** | ${PY_CI}s |" | |
| } >> "$GITHUB_STEP_SUMMARY" |