feat: merge-train/spartan

.github/workflows/deploy-irm.yml

@@ -6,7 +6,7 @@ on:
   workflow_call:
     inputs:
       network:
-        description: "Network that IRM will be monitoring e.g. testnet, staging-ignition..."
+        description: "Network that IRM will be monitoring e.g. testnet..."
         required: true
         type: string
         default: "testnet"
@@ -41,7 +41,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        description: "Network that IRM will be monitoring e.g. testnet, staging-ignition..."
+        description: "Network that IRM will be monitoring e.g. testnet..."
         required: true
         type: string
         default: "testnet"

.github/workflows/deploy-network.yml

@@ -38,10 +38,15 @@ on:
         description: "Source tag that triggered this deploy"
         required: false
         type: string
+      notify_on_failure:
+        description: "Whether this workflow should send its own failure notification"
+        required: false
+        type: boolean
+        default: true
   workflow_dispatch:
     inputs:
       network:
-        description: "Network to deploy (e.g., staging-public, staging-ignition, testnet, next-net)"
+        description: "Network to deploy (e.g., staging-public, testnet, next-net)"
         required: true
         type: choice
         options:
@@ -74,6 +79,11 @@ on:
         description: "Source tag that triggered this deploy"
         required: false
         type: string
+      notify_on_failure:
+        description: "Whether this workflow should send its own failure notification"
+        required: false
+        type: boolean
+        default: true
 
 concurrency:
   group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.aztec_docker_image || inputs.semver }}-${{ github.ref || github.ref_name }}
@@ -242,7 +252,7 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Notify Slack and dispatch ClaudeBox on failure
-        if: failure()
+        if: failure() && inputs.notify_on_failure
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}

.github/workflows/deploy-staging-public.yml

@@ -1,10 +1,6 @@
 name: Deploy to staging-public
 
 on:
-  workflow_run:
-    workflows: ["Nightly Release Tag (v4-next)"]
-    types:
-      - completed
   workflow_dispatch:
     inputs:
       tag:

.github/workflows/ensure-funded-environment.yml

@@ -6,7 +6,7 @@ on:
   workflow_call:
     inputs:
       environment:
-        description: 'Environment to fund (e.g., staging-public, next-net, staging-ignition, testnet)'
+        description: 'Environment to fund (e.g., staging-public, next-net, testnet)'
         required: true
         type: string
       low_watermark:
@@ -32,7 +32,6 @@ on:
         options:
         - staging-public
         - next-net
-        - staging-ignition
         - testnet
         - devnet
         - tps-scenario

.github/workflows/metrics-deploy.yml

@@ -100,7 +100,6 @@ jobs:
       GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME: ${{ inputs.grafana_dashboard_password_secret_name }}
       SLACK_WEBHOOK_URL_SECRET_NAME: slack-webhook-url
       SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME: slack-webhook-staging-public-url
-      SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME: slack-webhook-staging-ignition-url
       SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME: slack-webhook-next-scenario-url
       SLACK_WEBHOOK_NEXT_NET_SECRET_NAME: slack-webhook-next-net-url
       SLACK_WEBHOOK_TESTNET_SECRET_NAME: slack-webhook-testnet-url
@@ -164,7 +163,6 @@ jobs:
             -var="GRAFANA_PASSWORD_SECRET_NAME=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_SECRET_NAME=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME }}" \
-            -var="SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_NET_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_NET_SECRET_NAME }}" \
             -lock=${{ inputs.respect_tf_lock }}
@@ -179,7 +177,6 @@ jobs:
             -var="GRAFANA_PASSWORD_SECRET_NAME=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_SECRET_NAME=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME }}" \
-            -var="SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_NET_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_NET_SECRET_NAME }}" \
             -out=tfplan \

.github/workflows/weekly-proving-bench.yml

@@ -2,7 +2,7 @@ name: Weekly Real Proving Benchmark
 
 on:
   schedule:
-    - cron: "0 6 * * 1"  # Every Monday at 6 AM UTC
+    - cron: "0 6 * * 1" # Every Monday at 6 AM UTC
   workflow_dispatch:
     inputs:
       nightly_tag:
@@ -15,8 +15,11 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  real-proving-benchmark:
+  select-image:
     runs-on: ubuntu-latest
+    outputs:
+      nightly_tag: ${{ steps.nightly-tag.outputs.nightly_tag }}
+      docker_image: ${{ steps.nightly-tag.outputs.docker_image }}
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -32,12 +35,15 @@ jobs:
             current_version=$(jq -r '."."' .release-please-manifest.json)
             nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)"
           fi
-          echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
+
+          docker_image="aztecprotocol/aztec:${nightly_tag}"
+          echo "nightly_tag=$nightly_tag" >> "$GITHUB_OUTPUT"
+          echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT"
           echo "Using nightly tag: $nightly_tag"
 
       - name: Check if Docker image exists
         run: |
-          DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+          DOCKER_IMAGE="${{ steps.nightly-tag.outputs.docker_image }}"
           echo "Checking if Docker image exists: $DOCKER_IMAGE"
           if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
             echo "Docker image exists: $DOCKER_IMAGE"
@@ -46,6 +52,53 @@ jobs:
             exit 1
           fi
 
+  deploy-real-proving-network:
+    needs: select-image
+    uses: ./.github/workflows/deploy-network.yml
+    with:
+      network: prove-n-tps-real
+      namespace: prove-n-tps-real
+      aztec_docker_image: ${{ needs.select-image.outputs.docker_image }}
+      ref: next
+      notify_on_failure: false
+    secrets: inherit
+
+  wait-for-first-l2-block:
+    needs: deploy-real-proving-network
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: next
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a
+        with:
+          install_components: gke-gcloud-auth-plugin
+
+      - name: Wait for first L2 block
+        env:
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+        run: |
+          cd spartan
+          ./bootstrap.sh wait_for_l2_block prove-n-tps-real
+
+  benchmark:
+    needs: wait-for-first-l2-block
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: next
+
       - name: Run real proving benchmarks
         timeout-minutes: 180
         env:
@@ -59,20 +112,9 @@ jobs:
           RUN_ID: ${{ github.run_id }}
           AWS_SHUTDOWN_TIME: 180
           NO_SPOT: 1
+          SKIP_NETWORK_DEPLOY: "1"
         run: |
-          ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
-
-      - name: Cleanup network resources
-        if: always()
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
-          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
-          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
-          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-          NO_SPOT: 1
-        run: ./.github/ci3.sh network-teardown prove-n-tps-real prove-n-tps-real
+          ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real
 
       - name: Download benchmarks
         if: always()
@@ -81,7 +123,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           if ./ci.sh gh-spartan-proving-bench; then
-            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
+            echo "ENABLE_DEPLOY_BENCH=1" >> "$GITHUB_ENV"
           fi
 
       - name: Upload benchmarks
@@ -100,13 +142,52 @@ jobs:
           fail-on-alert: false
           max-items-in-chart: 100
 
+  cleanup:
+    if: always()
+    needs:
+      - select-image
+      - deploy-real-proving-network
+      - wait-for-first-l2-block
+      - benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: next
+
+      - name: Cleanup network resources
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          NO_SPOT: 1
+        run: ./.github/ci3.sh network-teardown prove-n-tps-real prove-n-tps-real
+
+  notify-failure:
+    if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }}
+    needs:
+      - select-image
+      - deploy-real-proving-network
+      - wait-for-first-l2-block
+      - benchmark
+      - cleanup
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: next
+
       - name: Notify Slack and dispatch ClaudeBox on failure
-        if: failure() && github.event_name != 'workflow_dispatch'
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
         run: |
-          TAG="${{ steps.nightly-tag.outputs.nightly_tag }}"
+          TAG="${{ needs.select-image.outputs.nightly_tag || 'unknown' }}"
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \
             "Weekly Real Proving Benchmark FAILED (nightly tag ${TAG}): <${RUN_URL}|View Run> (🤖)" \

.test_patterns.yml

@@ -164,6 +164,10 @@ tests:
     error_regex: "Error: Timeout of 2000ms exceeded"
     owners:
       - *martin
+  - regex: "kv-store/.*store\\.test"
+    error_regex: "guards against too many cursors"
+    owners:
+      - *palla
   - regex: "ethereum/src/test/tx_delayer.test.ts"
     error_regex: "delays a transaction until a given L1 timestamp"
     owners:
@@ -172,6 +176,18 @@ tests:
     error_regex: "ContractFunctionExecutionError: The contract function"
     owners:
       - *mitch
+  # Under proposer pipelining each validator votes in its own slot and the votes
+  # don't aggregate into the same round, so the slashing quorum (3) is never
+  # reached within the 414s budget; the test consistently times out at the docker
+  # outer 600s (exit 124). The publisher refactor lands all vote-offenses tx's
+  # on L1 successfully — voteCount on the slasher proposer simply stays at 1
+  # per round. This is a slashing-payload aggregation issue independent of
+  # publisher work; skip until the slashing team addresses it separately.
+  - regex: "e2e_p2p/valid_epoch_pruned_slash.test.ts"
+    skip: true
+    owners:
+      - *mitch
+      - *palla
   - regex: "archiver/src/archiver/archiver.test.ts"
     error_regex: "Received number of calls: 1"
     owners:
@@ -185,14 +201,6 @@ tests:
       - *phil
       - *palla
 
-  # http://ci.aztec-labs.com/64a972aafaa40dd0
-  # ProvingBroker › Retries › does not retry if job is stale — kv-store closes
-  # before the broker's final reportProvingJobError write lands.
-  - regex: "prover-client/src/proving_broker/proving_broker.test.ts"
-    error_regex: "does not retry if job is stale|Store is closed"
-    owners:
-      - *alex
-
   # Nightly GKE tests
   - regex: "spartan/bootstrap.sh"
     owners:

aztec-up/bootstrap.sh

@@ -20,6 +20,9 @@ function build {
   echo
 
   # Create Verdaccio config.
+  # publish.allow_offline lets verdaccio accept publishes when the npmjs
+  # uplink is briefly unreachable, instead of returning 503. We never want
+  # the upstream existence check to gate these local fake-publishes.
   cat > /tmp/verdaccio-config.yaml <<EOF
 storage: $PWD/verdaccio-storage
 max_body_size: 1000mb
@@ -28,7 +31,23 @@ uplinks:
   npmjs:
     url: https://registry.npmjs.org/
 
+publish:
+  allow_offline: true
+
 packages:
+  # @aztec/viem is a third-party fork published to npm, not built locally.
+  # Match it before @aztec/* so it falls through to the npmjs uplink.
+  "@aztec/viem":
+    access: \$all
+    publish: \$all
+    unpublish: \$all
+    proxy: npmjs
+
+  "@aztec/*":
+    access: \$all
+    publish: \$all
+    unpublish: \$all
+
   "@*/*":
     access: \$all
     publish: \$all

aztec-up/test/amm_flow.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!

aztec-up/test/basic_install.sh

@@ -10,6 +10,7 @@ echo
 
 export LOG_LEVEL=silent
 export PXE_PROVER=none
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
 
 # Start local network and wait for port to open.
 aztec start --local-network &

aztec-up/test/bridge_and_claim.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!