test(e2e): migrate block_building, multi_validator_node, publisher_funding, invalid_checkpoint_proposal to pipelining

yarn-project/end-to-end/src/e2e_block_building.test.ts

@@ -562,11 +562,7 @@ describe('e2e_block_building', () => {
     // The culprit is a nullifier not being cleared up from world state during block building if a tx fails processing,
     // which translates in an incorrect end state for world state. We can easily detect this by checking whether the nullifier
     // tree next available leaf index is a multiple of 64.
-    // TODO(kill-non-pipelined): under pipelining, an AVM failure mid-block triggers a
-    // `DELETE_FORK failed: Fork not found` loop in world-state and the sequencer's publisher
-    // is left in `Transaction sending is interrupted`. This needs a source-level fix in the
-    // pipelined checkpoint job's fork-cleanup path; the test invariant is still relevant.
-    it.skip('clears up all nullifiers if tx processing fails', async () => {
+    it('clears up all nullifiers if tx processing fails', async () => {
       const context = await setup(1, { ...PIPELINING_SETUP_OPTS, minTxsPerBlock: 1, numberOfInitialFundedAccounts: 1 });
       ({
         teardown,
@@ -598,23 +594,28 @@ describe('e2e_block_building', () => {
       const txHashResults = await Promise.all(batches.map(batch => batch.send({ from: ownerAddress, wait: NO_WAIT })));
       const txHashes = txHashResults.map(({ txHash }) => txHash);
       logger.warn(`Sent two txs to test contract`, { txs: txHashes.map(hash => hash.toString()) });
-      await Promise.race(txHashes.map(txHash => waitForTx(aztecNode, txHash, { timeout: 60 })));
+      // Use Promise.any (not Promise.race): exactly one of the two txs will be dropped (the one that hits
+      // the fake AVM error in tx processing), so the dropped-tx rejection would settle Promise.race first.
+      // We want the first *successful* mine.
+      const minedTxHash = await Promise.any(
+        txHashes.map(async txHash => {
+          await waitForTx(aztecNode, txHash, { timeout: 60 });
+          return txHash;
+        }),
+      );
 
-      logger.warn(`At least one tx has been mined`);
-      const lastBlock = (await context.aztecNode.getBlockData('latest'))?.header;
-      expect(lastBlock).toBeDefined();
+      logger.warn(`At least one tx has been mined`, { minedTxHash: minedTxHash.toString() });
+      const minedReceipt = await aztecNode.getTxReceipt(minedTxHash);
+      const block = await context.aztecNode.getBlock(minedReceipt.blockNumber!);
+      expect(block).toBeDefined();
 
-      logger.warn(`Latest block is ${lastBlock!.getBlockNumber()}`, { state: lastBlock?.state.partial });
-      const nextNullifierIndex = lastBlock!.state.partial.nullifierTree.nextAvailableLeafIndex;
+      logger.warn(`Mined block is ${block!.header.getBlockNumber()}`, { state: block!.header.state.partial });
+      const nextNullifierIndex = block!.header.state.partial.nullifierTree.nextAvailableLeafIndex;
       expect(nextNullifierIndex % 64).toEqual(0);
     });
   });
 
-  // TODO(kill-non-pipelined): reorg path under pipelined sequencer hangs to wallclock after
-  // `advanceToNextEpoch` + `markAsProven`. The world-state hits a `DELETE_FORK failed: Fork not
-  // found` loop and PXE catch-up never completes. Needs source-level fix in the pipelined
-  // checkpoint job's fork-cleanup path on prune.
-  describe.skip('reorgs', () => {
+  describe('reorgs', () => {
     let contract: StatefulTestContract;
     let cheatCodes: CheatCodes;
     let ownerAddress: AztecAddress;
@@ -630,18 +631,20 @@ describe('e2e_block_building', () => {
         cheatCodes,
         watcher,
         accounts: [ownerAddress],
-      } = await setup(1, { ...PIPELINING_SETUP_OPTS }));
+      } = await setup(1, { ...PIPELINING_SETUP_OPTS, minTxsPerBlock: 1 }));
 
       ({ contract } = await StatefulTestContract.deploy(wallet, ownerAddress, 1).send({ from: ownerAddress }));
       initialBlockNumber = await aztecNode.getBlockNumber();
       logger.info(`Stateful test contract deployed at ${contract.address}`);
 
       await cheatCodes.rollup.advanceToNextEpoch();
 
+      // Mark all blocks up to the current pending tip as proven so the contract-deployment block
+      // is anchored against a proven checkpoint. The e2e fixture's AnvilTestWatcher does NOT
+      // auto-prove under interval mining (only under automining), so we must drive proven manually.
+      await cheatCodes.rollup.markAsProven();
       const bn = await aztecNode.getBlockNumber();
-      while ((await aztecNode.getBlockNumber('proven')) < bn) {
-        await sleep(1000);
-      }
+      await retryUntil(async () => (await aztecNode.getBlockNumber('proven')) >= bn, 'wait-proven', 60, 1);
 
       watcher.setIsMarkingAsProven(false);
     });

yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts

@@ -1,12 +1,12 @@
+import type { InitialAccountData } from '@aztec/accounts/testing';
 import type { Archiver } from '@aztec/archiver';
 import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses';
-import { waitForProven } from '@aztec/aztec.js/contracts';
 import { ContractDeployer } from '@aztec/aztec.js/deployment';
 import { Fr } from '@aztec/aztec.js/fields';
 import type { Logger } from '@aztec/aztec.js/log';
 import type { AztecNode } from '@aztec/aztec.js/node';
-import type { Wallet } from '@aztec/aztec.js/wallet';
 import type { CheatCodes } from '@aztec/aztec/testing';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
 import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config';
@@ -19,22 +19,30 @@ import { retryUntil } from '@aztec/foundation/retry';
 import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi';
 import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest';
 import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p';
+import { TxStatus } from '@aztec/stdlib/tx';
 
+import { jest } from '@jest/globals';
 import { getContract } from 'viem';
 import { privateKeyToAccount } from 'viem/accounts';
 
+import { PIPELINING_SETUP_OPTS } from '../fixtures/fixtures.js';
 import { getPrivateKeyFromIndex, setup } from '../fixtures/utils.js';
+import type { TestWallet } from '../test-wallet/test_wallet.js';
 
 const VALIDATOR_COUNT = 5;
 const COMMITTEE_SIZE = VALIDATOR_COUNT - 2;
 const PUBLISHER_COUNT = 2;
 
 describe('e2e_multi_validator_node', () => {
+  // Each test starts its own multi-validator network and waits for checkpointed L2 transactions.
+  jest.setTimeout(15 * 60 * 1000);
+
   let initialValidatorPrivateKeys: `0x${string}`[];
   let validatorAddresses: `0x${string}`[];
   let teardown: () => Promise<void>;
-  let wallet: Wallet;
+  let wallet: TestWallet;
   let ownerAddress: AztecAddress;
+  let initialFundedAccounts: InitialAccountData[];
   let aztecNode: AztecNode;
   let config: AztecNodeConfig;
   let logger: Logger;
@@ -67,26 +75,22 @@ describe('e2e_multi_validator_node', () => {
     });
     const { aztecSlotDuration: _aztecSlotDuration } = getL1ContractsConfigEnvVars();
 
-    ({
-      teardown,
-      logger,
-      wallet,
-      accounts: [ownerAddress],
-      aztecNode,
-      config,
-      deployL1ContractsValues,
-      cheatCodes,
-    } = await setup(1, {
-      initialValidators,
-      aztecTargetCommitteeSize: COMMITTEE_SIZE,
-      sequencerPublisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)),
-      minTxsPerBlock: 1,
-      archiverPollingIntervalMS: 200,
-      sequencerPollingIntervalMS: 200,
-      worldStateBlockCheckIntervalMS: 200,
-      blockCheckIntervalMS: 200,
-      startProverNode: true,
-    }));
+    ({ teardown, logger, wallet, initialFundedAccounts, aztecNode, config, deployL1ContractsValues, cheatCodes } =
+      await setup(
+        1,
+        {
+          ...PIPELINING_SETUP_OPTS,
+          initialValidators,
+          aztecTargetCommitteeSize: COMMITTEE_SIZE,
+          sequencerPublisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)),
+          archiverPollingIntervalMS: 200,
+          sequencerPollingIntervalMS: 200,
+          worldStateBlockCheckIntervalMS: 200,
+          blockCheckIntervalMS: 200,
+          skipAccountDeployment: true,
+        },
+        { syncChainTip: 'checkpointed' },
+      ));
 
     rollup = new RollupContract(
       deployL1ContractsValues.l1Client,
@@ -109,16 +113,34 @@ describe('e2e_multi_validator_node', () => {
     await teardown();
   });
 
+  const deployOwnerAccount = async () => {
+    const accountData = initialFundedAccounts[0];
+    const accountManager = await wallet.createSchnorrAccount(
+      accountData.secret,
+      accountData.salt,
+      accountData.signingKey,
+    );
+    const deployMethod = await accountManager.getDeployMethod();
+    await deployMethod.send({
+      from: NO_FROM,
+      wait: {
+        waitForStatus: TxStatus.CHECKPOINTED,
+        timeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration,
+      },
+    });
+    await wallet.sync();
+    ownerAddress = accountManager.address;
+  };
+
   it('should build blocks & attest with multiple validator keys', async () => {
+    await deployOwnerAccount();
+
     const deployer = new ContractDeployer(artifact, wallet);
 
     logger.info(`Deploying contract from ${ownerAddress}`);
     const { receipt: tx } = await deployer
       .deploy([ownerAddress, 1], { salt: new Fr(BigInt(1)) })
       .send({ from: ownerAddress });
-    await waitForProven(aztecNode, tx, {
-      provenTimeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration,
-    });
     expect(tx.blockNumber).toBeDefined();
 
     const dataStore = (aztecNode as AztecNodeService).getBlockSource() as Archiver;
@@ -166,6 +188,7 @@ describe('e2e_multi_validator_node', () => {
         BigInt(await cheatCodes.rollup.getEpoch()) + BigInt(config.lagInEpochsForValidatorSet + 1),
       ),
     );
+    await deployOwnerAccount();
 
     // check that the committee is undefined
     const committee = await rollup.getCurrentEpochCommittee();
@@ -177,9 +200,6 @@ describe('e2e_multi_validator_node', () => {
     const { receipt: tx } = await deployer
       .deploy([ownerAddress, 1], { salt: new Fr(BigInt(1)) })
       .send({ from: ownerAddress });
-    await waitForProven(aztecNode, tx, {
-      provenTimeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration,
-    });
     expect(tx.blockNumber).toBeDefined();
 
     const dataStore = (aztecNode as AztecNodeService).getBlockSource() as Archiver;
@@ -196,8 +216,13 @@ describe('e2e_multi_validator_node', () => {
 
     expect(attestations.length).toBeGreaterThanOrEqual((COMMITTEE_SIZE * 2) / 3 + 1);
 
-    const signers = attestations.map(att => att.getSender()!.toString());
+    const signers = attestations.map(att => att.getSender()!.toString().toLowerCase());
+    const validatorAddressesLower = validatorAddresses.map(a => a.toLowerCase());
+    expect(signers.every(s => validatorAddressesLower.includes(s))).toBe(true);
 
-    expect(signers).toEqual(expect.arrayContaining(validatorAddresses.slice(0, COMMITTEE_SIZE)));
+    const committeeAtCheckpoint = await rollup.getCommitteeAt(publishedCheckpoint.checkpoint.header.timestamp);
+    expect(committeeAtCheckpoint?.length).toBe(COMMITTEE_SIZE);
+    const committeeAtCheckpointLower = committeeAtCheckpoint!.map(a => a.toString().toLowerCase());
+    expect(signers.every(s => committeeAtCheckpointLower.includes(s))).toBe(true);
   });
 });

yarn-project/end-to-end/src/e2e_publisher_funding_multi.test.ts

@@ -17,6 +17,7 @@ import { join } from 'path';
 import { type Hex, parseEther } from 'viem';
 import { privateKeyToAccount } from 'viem/accounts';
 
+import { PIPELINING_SETUP_OPTS } from './fixtures/fixtures.js';
 import { getPrivateKeyFromIndex, setup } from './fixtures/utils.js';
 
 // Key indices from the test MNEMONIC (all pre-funded by Anvil):
@@ -89,6 +90,7 @@ describe('e2e_publisher_funding_multi', () => {
       sequencer: sequencerClient,
       ethCheatCodes,
     } = await setup(0, {
+      ...PIPELINING_SETUP_OPTS,
       initialValidators,
       keyStoreDirectory,
       publisherFundingThreshold: FUNDING_THRESHOLD,
@@ -156,9 +158,11 @@ describe('e2e_publisher_funding_multi', () => {
     // Funder should have sent 2 * FUNDING_AMOUNT plus gas costs (single multicall)
     expect(funderSpent).toBeGreaterThanOrEqual(2n * FUNDING_AMOUNT);
 
-    // Second round: after funding, publishers are above threshold. The active publisher will
-    // spend gas publishing blocks and eventually drop below threshold again, triggering re-funding
-    // for that one publisher.
+    // Second round: deterministically drop one publisher below threshold after the first refill.
+    // Waiting for organic gas depletion is brittle under pipelined publisher rotation: the exact
+    // publisher cadence and L1 gas burn vary enough that the balance may not cross the threshold
+    // before the test timeout, even though the periodic funding loop is healthy.
+    await ethCheatCodes.setBalance(publisher1Address, LOW_BALANCE);
     const funderBalanceBefore2 = await ethCheatCodes.getBalance(funderAddress);
     logger.info(`Waiting for second funding round`);
 

yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts

@@ -221,7 +221,8 @@ describe('e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', () => {
         aztecSlotDuration: AZTEC_SLOT_DURATION,
         aztecTargetCommitteeSize: COMMITTEE_SIZE,
         aztecProofSubmissionEpochs: 1024,
-        enableProposerPipelining: false,
+        enableProposerPipelining: true,
+        inboxLag: 2,
         mockGossipSubNetwork: true,
         slashingQuorum: SLASHING_QUORUM,
         slashingRoundSizeInEpochs: SLASHING_ROUND_SIZE / AZTEC_EPOCH_DURATION,
@@ -264,7 +265,7 @@ describe('e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', () => {
       {
         ...t.ctx.aztecNodeConfig,
         dontStartSequencer: true,
-        enableProposerPipelining: false,
+        enableProposerPipelining: true,
         slashBroadcastedInvalidCheckpointProposalPenalty: slashingUnit,
         slashSelfAllowed: true,
       },