BenchmarkComment.yml

name: Benchmark Comment

on:
  workflow_run:
    workflows: ["Benchmark"]
    types: [completed]

permissions:
  contents: read
  pull-requests: write

jobs:
  comment:
    name: Post Benchmark Comment
    runs-on: ubuntu-latest
    if: >
      github.event.workflow_run.conclusion == 'success' &&
      github.event.workflow_run.event == 'pull_request'

    steps:
      - name: Download benchmark results
        uses: actions/download-artifact@v4
        with:
          name: benchmark-results
          run-id: ${{ github.event.workflow_run.id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Get PR info
        id: pr-info
        run: |
          if [ -f pr_number ]; then
            echo "pr_number=$(cat pr_number)" >> $GITHUB_OUTPUT
            echo "pr_sha=$(cat pr_head_sha)" >> $GITHUB_OUTPUT
          else
            echo "::error::PR metadata not found in artifact"
            exit 1
          fi

      - name: Generate benchmark comment
        env:
          PR_SHA: ${{ steps.pr-info.outputs.pr_sha }}
        run: |
          PR_SHORT="${PR_SHA:0:8}"

          {
            echo "<!-- benchmark-comment -->"
            echo "# FastInterpolations.jl Benchmarks"
            echo ""

            if [ -f regression_report.json ]; then
              # ── Rich report from regression verification ──
              TOTAL=$(jq '.summary.total' regression_report.json)
              FLAGGED=$(jq '.summary.initially_flagged' regression_report.json)
              CONFIRMED=$(jq '.summary.confirmed_regressions' regression_report.json)
              RERUN_N=$(jq '.thresholds.rerun_n' regression_report.json)
              IMM_TH=$(jq '.thresholds.immediate' regression_report.json)
              GRAD_TH=$(jq '.thresholds.gradual' regression_report.json)

              # All benchmarks table (collapsed)
              echo "<details>"
              echo "<summary>All benchmarks (${TOTAL} total, click to expand)</summary>"
              echo ""
              echo "| Benchmark | Current: ${PR_SHORT} | Previous | Imm. Ratio | Grad. Ratio |"
              echo "|-----------|---------------------:|----------:|-----------:|------------:|"
              jq -r '.benchmarks[] |
                "| `\(.name)` | `\(.value)` ns | `\(.prev_value // "-")` ns | `\(.ratio_immediate // "-")` | `\(.ratio_gradual // "-")` |"
              ' regression_report.json
              echo ""
              echo "</details>"

              # Verification summary
              if [ "$CONFIRMED" -gt 0 ]; then
                echo ""
                echo "## :warning: **Performance Regression Confirmed** :warning:"
                echo ""
                echo "After re-running ${FLAGGED} flagged benchmark(s) ${RERUN_N} time(s), **${CONFIRMED} regression(s) confirmed**."
                echo ""
                echo "| Benchmark | Current | Previous | Imm. Ratio | Grad. Ratio | Tier |"
                echo "|-----------|--------:|----------:|-----------:|------------:|------|"
                jq -r '.benchmarks[] | select(.regression == true) |
                  "| `\(.name)` | `\(.value)` ns | `\(.prev_value // "-")` ns | `\(.ratio_immediate // "-")` | `\(.ratio_gradual // "-")` | \(if .ratio_immediate != null and .ratio_immediate > '"$IMM_TH"' then (if .ratio_gradual != null and .ratio_gradual > '"$GRAD_TH"' then "both" else "immediate" end) else "gradual" end) |"
                ' regression_report.json
                echo ""
                echo "> Thresholds: immediate > ${IMM_TH}x (vs latest master), gradual > ${GRAD_TH}x (vs sliding window)"
              elif [ "$FLAGGED" -gt 0 ]; then
                echo ""
                echo "> :white_check_mark: **${FLAGGED} benchmark(s) were flagged but verified as CI noise after ${RERUN_N} re-run(s)**"
              else
                echo ""
                echo "> :white_check_mark: No regressions detected"
              fi

            elif [ -f benchmark_flat.json ]; then
              # ── Fallback: no regression report (no baseline available) ──
              echo "<details>"
              echo "<summary>All benchmarks (click to expand)</summary>"
              echo ""
              echo "| Benchmark | Current: ${PR_SHORT} |"
              echo "|-----------|---------------------:|"
              jq -r '.[] | "| `\(.name)` | `\(.value)` \(.unit) |"' benchmark_flat.json
              echo ""
              echo "</details>"
              echo ""
              echo "> No baseline data available for comparison (first benchmark run?)"

            else
              echo "> No benchmark data found in artifacts"
            fi

            echo ""
            echo "This comment was automatically generated by [Benchmark workflow](https://github.com/${{ github.repository }}/actions?query=workflow%3ABenchmark)."
          } > comment_body.md

      - name: Post or update PR comment
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }}
        run: |
          COMMENT_TAG="<!-- benchmark-comment -->"
          EXISTING=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
            --jq ".[] | select(.body | contains(\"${COMMENT_TAG}\")) | .id" | head -1)

          if [ -n "$EXISTING" ]; then
            gh api "repos/${{ github.repository }}/issues/comments/${EXISTING}" \
              -X PATCH -F body=@comment_body.md
          else
            gh pr comment "$PR_NUMBER" --repo "${{ github.repository }}" --body-file comment_body.md
          fi