php-sdk/.github/workflows/conformance-weekly.yaml at main · modelcontextprotocol/php-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
name: conformance-weekly

# Runs the MCP conformance suite weekly against the latest
# @modelcontextprotocol/conformance release. The on:pull_request pipeline
# pins to whatever version is available at PR time; this schedule catches
# upstream releases that add scenarios between PRs.
#
# It also scores each run and publishes the client/server pass-rate as
# shields.io endpoint JSON to the orphan `badges` branch (consumed by the
# README); that branch is created on the first run.
on:
  schedule:
    - cron: '0 6 * * 1' # Mondays 06:00 UTC
  workflow_dispatch:

permissions:
  contents: write
  issues: write

jobs:
  server:
    name: conformance / server (latest)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-node@v6
        with:
          node-version: '22'
      - run: composer install --prefer-dist --no-progress --no-interaction
      - name: Start conformance server
        run: |
          mkdir -p tests/Conformance/sessions tests/Conformance/logs
          chmod -R 777 tests/Conformance/sessions tests/Conformance/logs
          docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d
          sleep 5
      - name: Run conformance tests
        working-directory: ./tests/Conformance
        run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml --output-dir results
      - name: Generate score badge
        if: always()
        run: php tests/Conformance/score.php server
      - name: Show docker logs on failure
        if: failure()
        run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs
      - name: Upload conformance results
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-server-results
          path: |
            tests/Conformance/logs
            tests/Conformance/results
      - name: Upload score badge
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: server-badge
          path: tests/Conformance/server-conformance.json

  client:
    name: conformance / client (latest)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: shivammathur/setup-php@v2
        with:
          php-version: '8.4'
          coverage: none
      - uses: actions/setup-node@v6
        with:
          node-version: '22'
      - run: composer install --prefer-dist --no-progress --no-interaction
      - run: mkdir -p tests/Conformance/logs
      - name: Run conformance tests
        working-directory: ./tests/Conformance
        run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml --output-dir results
      - name: Generate score badge
        if: always()
        run: php tests/Conformance/score.php client
      - name: Upload conformance results
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-client-results
          path: |
            tests/Conformance/logs
            tests/Conformance/results
      - name: Upload score badge
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: client-badge
          path: tests/Conformance/client-conformance.json

  notify:
    name: Open issue on failure
    runs-on: ubuntu-latest
    needs: [server, client]
    if: failure() && github.event_name == 'schedule'
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GH_REPO: ${{ github.repository }}
      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
    steps:
      - name: File or comment tracking issue
        run: |
          existing=$(gh issue list --label conformance-weekly --state open --json number --jq '.[0].number // empty')
          if [ -n "$existing" ]; then
            gh issue comment "$existing" --body "New failure on $(date -u +%FT%TZ): $RUN_URL"
          else
            gh issue create \
              --title '[conformance] Weekly conformance run failed' \
              --label conformance-weekly \
              --body "Weekly conformance against \`@modelcontextprotocol/conformance@latest\` failed.

          - Run: $RUN_URL
          - Triggered: $(date -u +%FT%TZ)

          Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline.yml\`."
          fi

  publish:
    name: Publish conformance badges
    runs-on: ubuntu-latest
    needs: [server, client]
    # Publish even when the suite regressed (the badge should reflect reality);
    # skip on forks, which cannot push the `badges` branch.
    if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/download-artifact@v8
        with:
          name: server-badge
          path: badges-in
      - uses: actions/download-artifact@v8
        with:
          name: client-badge
          path: badges-in
      - name: Publish to badges branch
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          if git ls-remote --exit-code --heads origin badges >/dev/null 2>&1; then
            git fetch origin badges
            git worktree add badges-wt badges
          else
            git worktree add --detach badges-wt
            git -C badges-wt checkout --orphan badges
            git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true
          fi

          cp badges-in/server-conformance.json badges-in/client-conformance.json badges-wt/

          cd badges-wt
          git add -A
          if git diff --cached --quiet; then
            echo "Conformance scores unchanged."
          else
            git commit -m "Update conformance score badges"
            git push origin badges
          fi