refactor!: optimizing AztecNode::getLogsByTags for new log sync algo

yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts

@@ -58,7 +58,15 @@ export class LogStore {
 
         const currentLogs = taggedLogs.get(tag.toString()) ?? [];
         currentLogs.push(
-          new TxScopedL2Log(txHash, dataStartIndexForTx, logIndex, block.number, blockHash, log).toBuffer(),
+          new TxScopedL2Log(
+            txHash,
+            dataStartIndexForTx,
+            logIndex,
+            block.number,
+            blockHash,
+            block.timestamp,
+            log,
+          ).toBuffer(),
         );
         taggedLogs.set(tag.toString(), currentLogs);
       });
@@ -69,7 +77,15 @@ export class LogStore {
 
         const currentLogs = taggedLogs.get(tag.toString()) ?? [];
         currentLogs.push(
-          new TxScopedL2Log(txHash, dataStartIndexForTx, logIndex, block.number, blockHash, log).toBuffer(),
+          new TxScopedL2Log(
+            txHash,
+            dataStartIndexForTx,
+            logIndex,
+            block.number,
+            blockHash,
+            block.timestamp,
+            log,
+          ).toBuffer(),
         );
         taggedLogs.set(tag.toString(), currentLogs);
       });

yarn-project/foundation/src/schemas/schemas.ts

@@ -58,6 +58,14 @@ export const schemas = {
       .max(2 ** 32 - 1),
   ),
 
+  /** Coerces input to UInt64. */
+  UInt64: z.union([z.bigint(), z.number(), z.string()]).pipe(
+    z.coerce
+      .bigint()
+      .min(0n)
+      .max(2n ** 64n - 1n),
+  ),
+
   /** Accepts a hex string as a Buffer32 type. */
   Buffer32: z.string().refine(isHex, 'Not a valid hex string').transform(Buffer32.fromString),
 

yarn-project/pxe/src/logs/log_service.test.ts

@@ -85,6 +85,7 @@ describe('LogService', () => {
           0,
           MIN_BLOCK_NUMBER_OF_A_LOG,
           L2BlockHash.random(),
+          0n,
           PrivateLog.random(tag.value),
         );
         logs[tag.toString()] = [log];
@@ -101,6 +102,7 @@ describe('LogService', () => {
         0,
         BlockNumber.ZERO,
         L2BlockHash.random(),
+        0n,
         PrivateLog.random(tag.value),
       );
       logs[tag.toString()].push(log);
@@ -118,6 +120,7 @@ describe('LogService', () => {
           0,
           blockNumber,
           L2BlockHash.random(),
+          0n,
           PrivateLog.random(tag.value),
         );
         logs[tag.toString()] = [log];
@@ -137,6 +140,7 @@ describe('LogService', () => {
           0,
           MAX_BLOCK_NUMBER_OF_A_LOG,
           L2BlockHash.random(),
+          0n,
           PrivateLog.random(tag.value),
         );
         logs[tag.toString()] = [log];
@@ -553,6 +557,7 @@ describe('LogService', () => {
         randomInt(100),
         BlockNumber(randomInt(100)),
         L2BlockHash.random(),
+        0n,
         log,
       );
 

yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts

@@ -35,13 +35,15 @@ describe('loadPrivateLogsForSenderRecipientPair', () => {
     return SiloedTag.compute(tag, app);
   }
 
-  function makeLog(blockHash: Fr, blockNumber: number, tag: Fr) {
+  // Move blockTimestamp before tag in args in makeLog
+  function makeLog(blockHash: Fr, blockNumber: number, blockTimestamp: bigint, tag: Fr) {
     return new TxScopedL2Log(
       TxHash.random(),
       0,
       0,
       BlockNumber(blockNumber),
       L2BlockHash.fromField(blockHash),
+      blockTimestamp,
       PrivateLog.random(tag),
     );
   }
@@ -54,7 +56,6 @@ describe('loadPrivateLogsForSenderRecipientPair', () => {
 
   beforeEach(async () => {
     aztecNode.getLogsByTags.mockReset();
-    aztecNode.getBlockHeaderByHash.mockReset();
     aztecNode.getL2Tips.mockReset();
     aztecNode.getBlockHeader.mockReset();
     taggingDataProvider = new NewRecipientTaggingDataProvider(await openTmpStore('test'));
@@ -103,18 +104,13 @@ describe('loadPrivateLogsForSenderRecipientPair', () => {
     aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => {
       return Promise.all(
         tags.map(async (t: Fr) =>
-          t.equals(logTag.value) ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logTag.value)] : [],
+          t.equals(logTag.value)
+            ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logBlockTimestamp, logTag.value)]
+            : [],
         ),
       );
     });
 
-    aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => {
-      if (hash.equals(await logBlockHeader.hash())) {
-        return logBlockHeader;
-      }
-      return undefined;
-    });
-
     const logs = await loadPrivateLogsForSenderRecipientPair(
       secret,
       app,
@@ -146,18 +142,13 @@ describe('loadPrivateLogsForSenderRecipientPair', () => {
     aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => {
       return Promise.all(
         tags.map(async (t: Fr) =>
-          t.equals(logTag.value) ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logTag.value)] : [],
+          t.equals(logTag.value)
+            ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logBlockTimestamp, logTag.value)]
+            : [],
         ),
       );
     });
 
-    aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => {
-      if (hash.equals(await logBlockHeader.hash())) {
-        return logBlockHeader;
-      }
-      return undefined;
-    });
-
     const logs = await loadPrivateLogsForSenderRecipientPair(
       secret,
       app,
@@ -203,24 +194,15 @@ describe('loadPrivateLogsForSenderRecipientPair', () => {
       return Promise.all(
         tags.map(async (t: Fr) => {
           if (t.equals(log1Tag.value)) {
-            return [makeLog(await log1BlockHeader.hash(), finalizedBlockNumber, log1Tag.value)];
+            return [makeLog(await log1BlockHeader.hash(), finalizedBlockNumber, log1BlockTimestamp, log1Tag.value)];
           } else if (t.equals(log2Tag.value)) {
-            return [makeLog(await log2BlockHeader.hash(), finalizedBlockNumber, log2Tag.value)];
+            return [makeLog(await log2BlockHeader.hash(), finalizedBlockNumber, log2BlockTimestamp, log2Tag.value)];
           }
           return [];
         }),
       );
     });
 
-    aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => {
-      if (hash.equals(await log1BlockHeader.hash())) {
-        return log1BlockHeader;
-      } else if (hash.equals(await log2BlockHeader.hash())) {
-        return log2BlockHeader;
-      }
-      return undefined;
-    });
-
     const logs = await loadPrivateLogsForSenderRecipientPair(
       secret,
       app,

yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts

@@ -88,16 +88,16 @@ export async function loadPrivateLogsForSenderRecipientPair(
 
   while (true) {
     // Get private logs with their block timestamps and corresponding tagging indexes
-    const logsWithTimestampsAndIndexes = await loadLogsForRange(secret, app, aztecNode, start, end, anchorBlockNumber);
+    const privateLogsWithIndexes = await loadLogsForRange(secret, app, aztecNode, start, end, anchorBlockNumber);
 
-    if (logsWithTimestampsAndIndexes.length === 0) {
+    if (privateLogsWithIndexes.length === 0) {
       break;
     }
 
-    logs.push(...logsWithTimestampsAndIndexes.map(({ log }) => log));
+    logs.push(...privateLogsWithIndexes.map(({ log }) => log));
 
     const { highestAgedIndex, highestFinalizedIndex } = findHighestIndexes(
-      logsWithTimestampsAndIndexes,
+      privateLogsWithIndexes,
       currentTimestamp,
       finalizedBlockNumber,
     );

yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts

@@ -9,27 +9,24 @@ import { findHighestIndexes } from './find_highest_indexes.js';
 describe('findHighestIndexes', () => {
   const currentTimestamp = BigInt(Math.floor(Date.now() / 1000));
 
-  function makeLog(blockNumber: number): TxScopedL2Log {
+  function makeLog(blockNumber: number, blockTimestamp: bigint): TxScopedL2Log {
     return new TxScopedL2Log(
       TxHash.random(),
       0,
       0,
       BlockNumber(blockNumber),
       L2BlockHash.random(),
+      blockTimestamp,
       PrivateLog.random(),
     );
   }
 
   it('returns undefined for highestAgedIndex when no logs are at least 24 hours old', () => {
     const finalizedBlockNumber = 10;
     const blockTimestamp = currentTimestamp - 1n; // not aged
-    const log = makeLog(5);
+    const log = makeLog(5, blockTimestamp);
 
-    const result = findHighestIndexes(
-      [{ log, blockTimestamp, taggingIndex: 3 }],
-      currentTimestamp,
-      finalizedBlockNumber,
-    );
+    const result = findHighestIndexes([{ log, taggingIndex: 3 }], currentTimestamp, finalizedBlockNumber);
 
     expect(result.highestAgedIndex).toBeUndefined();
     expect(result.highestFinalizedIndex).toBe(3);
@@ -38,13 +35,9 @@ describe('findHighestIndexes', () => {
   it('returns undefined for highestFinalizedIndex when no logs are in finalized blocks', () => {
     const finalizedBlockNumber = 5;
     const blockTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION);
-    const log = makeLog(10); // block 10 > finalizedBlockNumber 5
+    const log = makeLog(10, blockTimestamp); // block 10 > finalizedBlockNumber 5
 
-    const result = findHighestIndexes(
-      [{ log, blockTimestamp, taggingIndex: 3 }],
-      currentTimestamp,
-      finalizedBlockNumber,
-    );
+    const result = findHighestIndexes([{ log, taggingIndex: 3 }], currentTimestamp, finalizedBlockNumber);
 
     expect(result.highestAgedIndex).toBe(3);
     expect(result.highestFinalizedIndex).toBeUndefined();
@@ -54,14 +47,14 @@ describe('findHighestIndexes', () => {
     const finalizedBlockNumber = 10;
     const blockTimestamp1 = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 1000n; // aged
     const blockTimestamp2 = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 500n; // aged
-    const log1 = makeLog(5);
-    const log2 = makeLog(6);
+    const log1 = makeLog(5, blockTimestamp1);
+    const log2 = makeLog(6, blockTimestamp2);
 
     const result = findHighestIndexes(
       [
-        { log: log1, blockTimestamp: blockTimestamp1, taggingIndex: 2 },
-        { log: log2, blockTimestamp: blockTimestamp2, taggingIndex: 5 },
-        { log: log1, blockTimestamp: blockTimestamp1, taggingIndex: 3 },
+        { log: log1, taggingIndex: 2 },
+        { log: log2, taggingIndex: 5 },
+        { log: log1, taggingIndex: 3 },
       ],
       currentTimestamp,
       finalizedBlockNumber,
@@ -74,15 +67,15 @@ describe('findHighestIndexes', () => {
   it('selects the highest index from multiple finalized logs', () => {
     const finalizedBlockNumber = 10;
     const blockTimestamp = currentTimestamp - 500n; // 500 seconds ago - not aged
-    const log1 = makeLog(5);
-    const log2 = makeLog(8);
-    const log3 = makeLog(10); // At finalized block number
+    const log1 = makeLog(5, blockTimestamp);
+    const log2 = makeLog(8, blockTimestamp);
+    const log3 = makeLog(10, blockTimestamp); // At finalized block number
 
     const result = findHighestIndexes(
       [
-        { log: log1, blockTimestamp, taggingIndex: 2 },
-        { log: log2, blockTimestamp, taggingIndex: 7 },
-        { log: log3, blockTimestamp, taggingIndex: 1 },
+        { log: log1, taggingIndex: 2 },
+        { log: log2, taggingIndex: 7 },
+        { log: log3, taggingIndex: 1 },
       ],
       currentTimestamp,
       finalizedBlockNumber,
@@ -99,11 +92,11 @@ describe('findHighestIndexes', () => {
     const recentTimestamp = currentTimestamp - 5000n; // Not aged
 
     const logs = [
-      { log: makeLog(5), blockTimestamp: veryOldTimestamp, taggingIndex: 1 }, // Aged, finalized
-      { log: makeLog(8), blockTimestamp: oldTimestamp, taggingIndex: 5 }, // Aged, finalized
-      { log: makeLog(10), blockTimestamp: recentTimestamp, taggingIndex: 8 }, // Not aged, finalized
-      { log: makeLog(15), blockTimestamp: oldTimestamp, taggingIndex: 12 }, // Aged, not finalized
-      { log: makeLog(20), blockTimestamp: recentTimestamp, taggingIndex: 15 }, // Not aged, not finalized
+      { log: makeLog(5, veryOldTimestamp), taggingIndex: 1 }, // Aged, finalized
+      { log: makeLog(8, oldTimestamp), taggingIndex: 5 }, // Aged, finalized
+      { log: makeLog(10, recentTimestamp), taggingIndex: 8 }, // Not aged, finalized
+      { log: makeLog(15, oldTimestamp), taggingIndex: 12 }, // Aged, not finalized
+      { log: makeLog(20, recentTimestamp), taggingIndex: 15 }, // Not aged, not finalized
     ];
 
     const result = findHighestIndexes(logs, currentTimestamp, finalizedBlockNumber);

yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts

@@ -5,15 +5,15 @@ import type { TxScopedL2Log } from '@aztec/stdlib/logs';
  * Finds the highest aged and the highest finalized tagging indexes.
  */
 export function findHighestIndexes(
-  logsWithTimestampsAndIndexes: Array<{ log: TxScopedL2Log; blockTimestamp: bigint; taggingIndex: number }>,
+  privateLogsWithIndexes: Array<{ log: TxScopedL2Log; taggingIndex: number }>,
   currentTimestamp: bigint,
   finalizedBlockNumber: number,
 ): { highestAgedIndex: number | undefined; highestFinalizedIndex: number | undefined } {
   let highestAgedIndex = undefined;
   let highestFinalizedIndex = undefined;
 
-  for (const { log, blockTimestamp, taggingIndex } of logsWithTimestampsAndIndexes) {
-    const ageInSeconds = currentTimestamp - blockTimestamp;
+  for (const { log, taggingIndex } of privateLogsWithIndexes) {
+    const ageInSeconds = currentTimestamp - log.blockTimestamp;
 
     if (
       ageInSeconds >= BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) &&