feat(ensindexer): leverage SWR caches to achieve goals related to LocalPonderClient

.changeset/purple-glasses-shave.md

@@ -0,0 +1,5 @@
+---
+"ensindexer": minor
+---
+
+Implements `LocalPonderClient` allowing interactions with Ponder app configuration and data.

apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts

@@ -15,16 +15,14 @@ import {
 
 import { buildENSIndexerPublicConfig } from "@/config/public";
 import { createCrossChainIndexingStatusSnapshotOmnichain } from "@/lib/indexing-status/build-index-status";
-import { buildOmnichainIndexingStatusSnapshot } from "@/lib/indexing-status-builder/omnichain-indexing-status-snapshot";
 import { getLocalPonderClient } from "@/lib/ponder-api-client";
 
 const app = new Hono();
-
-// Calling `getLocalPonderClient` at the top level to initialize
-// the singleton client instance on app startup.
-// This ensures that the client is ready to use when handling requests,
-// and allows us to catch initialization errors early.
-getLocalPonderClient();
+// Get the local Ponder Client instance
+// Note that the client initialization is designed to be non-blocking,
+// by implementing internal SWR caches with proactive revalidation to
+// load necessary data for the client state.
+const localPonderClient = getLocalPonderClient();
 
 // include ENSIndexer Public Config endpoint
 app.get("/config", async (c) => {
@@ -42,10 +40,7 @@ app.get("/indexing-status", async (c) => {
   let omnichainSnapshot: OmnichainIndexingStatusSnapshot | undefined;
 
   try {
-    const localPonderClient = await getLocalPonderClient();
-    const chainsIndexingMetadata = await localPonderClient.chainsIndexingMetadata();
-
-    omnichainSnapshot = buildOmnichainIndexingStatusSnapshot(chainsIndexingMetadata);
+    omnichainSnapshot = await localPonderClient.getOmnichainIndexingStatusSnapshot();
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : "Unknown error";
     console.error(

apps/ensindexer/ponder/src/api/lib/cache/ponder-client.cache.ts

@@ -0,0 +1,66 @@
+import config from "@/config";
+
+import { type Duration, SWRCache } from "@ensnode/ensnode-sdk";
+import {
+  PonderClient,
+  type PonderIndexingMetrics,
+  type PonderIndexingStatus,
+} from "@ensnode/ponder-sdk";
+
+/**
+ * Result of the Ponder Client cache.
+ */
+export interface PonderClientCacheResult {
+  ponderIndexingMetrics: PonderIndexingMetrics;
+  ponderIndexingStatus: PonderIndexingStatus;
+}
+
+/**
+ * SWR Cache for Ponder Client data
+ */
+export type PonderClientCache = SWRCache<PonderClientCacheResult>;
+
+const ponderClient = new PonderClient(config.ensIndexerUrl);
+
+/**
+ * Cache for Ponder Client data
+ *
+ * In case of using multiple Ponder API endpoints, it is optimal for data
+ * consistency to call all endpoints at once. This cache loads both
+ * Ponder Indexing Metrics and Ponder Indexing Status together, and provides
+ * them as a single cached result. This way, we ensure that the metrics and
+ * status data are always from the same point in time, and avoid potential
+ * inconsistencies that could arise if they were loaded separately.
+ *
+ * Ponder Client may sometimes fail to load data, i.e. due to network issues.
+ * The cache is designed to be resilient to loading failures, and will keep data
+ * in the cache indefinitely until it can be successfully reloaded.
+ * See `ttl` option below.
+ *
+ * Ponder Indexing Metrics and Ponder Indexing Status can both change frequently,
+ * so the cache is designed to proactively revalidate data to ensure freshness.
+ * See `proactiveRevalidationInterval` option below.
+ *
+ * Note, that Ponder app needs a while at startup to populate indexing metrics,
+ * and indexing status, so a few of the initial attempts to load this cache may
+ * fail until required data is made available by the Ponder app.
+ */
+export const ponderClientCache = new SWRCache({
+  fn: async function loadPonderClientCache() {
+    try {
+      const [ponderIndexingMetrics, ponderIndexingStatus] = await Promise.all([
+        ponderClient.metrics(),
+        ponderClient.status(),
+      ]);
+
+      return { ponderIndexingMetrics, ponderIndexingStatus };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error(`[PonderClientCache]: an error occurred while loading data: ${errorMessage}`);
+
+      throw new Error(`Failed to load Ponder Client cache: ${errorMessage}`);
+    }
+  },
+  ttl: Number.POSITIVE_INFINITY,
+  proactiveRevalidationInterval: 10 satisfies Duration,
+}) satisfies PonderClientCache;

apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts

@@ -149,7 +149,7 @@ export function buildChainsBlockrange(
     //      ponderSource for that chain has its respective `endBlock` defined.
     const isEndBlockForChainAllowed = chainEndBlocks.length === chainStartBlocks.length;
 
-    // 3.b) Get the highest endBLock for the chain.
+    // 3.b) Get the highest endBlock for the chain.
     const chainHighestEndBlock =
       isEndBlockForChainAllowed && chainEndBlocks.length > 0
         ? Math.max(...chainEndBlocks)

apps/ensindexer/ponder/src/api/lib/chains-indexing-metadata-dynamic.ts

@@ -0,0 +1,48 @@
+import type { ChainId, PonderIndexingMetrics, PonderIndexingStatus } from "@ensnode/ponder-sdk";
+
+import type { ChainIndexingMetadataDynamic } from "@/lib/indexing-status-builder/chain-indexing-metadata";
+
+/**
+ * Build a map of chain ID to its dynamic indexing metadata.
+ *
+ * The dynamic metadata is based on the current indexing metrics and status
+ * of the chain, which can change over time as the chain is being indexed.
+ *
+ * @param indexedChainIds A set of chain IDs that are being indexed.
+ * @param ponderIndexingMetrics The current indexing metrics for all chains.
+ * @param ponderIndexingStatus The current indexing status for all chains.
+ *
+ * @returns A map of chain ID to its dynamic indexing metadata.
+ *
+ * @throws Error if any invariants are violated.
+ */
+export function buildChainsIndexingMetadataDynamic(
+  indexedChainIds: Set<ChainId>,
+  ponderIndexingMetrics: PonderIndexingMetrics,
+  ponderIndexingStatus: PonderIndexingStatus,
+): Map<ChainId, ChainIndexingMetadataDynamic> {
+  const chainsIndexingMetadataDynamic = new Map<ChainId, ChainIndexingMetadataDynamic>();
+
+  for (const chainId of indexedChainIds) {
+    const chainIndexingMetrics = ponderIndexingMetrics.chains.get(chainId);
+    const chainIndexingStatus = ponderIndexingStatus.chains.get(chainId);
+
+    // Invariants: indexing metrics and indexing status must exist in proper state for the indexed chain.
+    if (!chainIndexingMetrics) {
+      throw new Error(`Indexing metrics must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (!chainIndexingStatus) {
+      throw new Error(`Indexing status must be available for indexed chain ID ${chainId}`);
+    }
+
+    const metadataDynamic = {
+      indexingMetrics: chainIndexingMetrics,
+      indexingStatus: chainIndexingStatus,
+    } satisfies ChainIndexingMetadataDynamic;
+
+    chainsIndexingMetadataDynamic.set(chainId, metadataDynamic);
+  }
+
+  return chainsIndexingMetadataDynamic;
+}

apps/ensindexer/ponder/src/api/lib/chains-indexing-metadata-immutable.ts

@@ -0,0 +1,114 @@
+import type { PublicClient } from "viem";
+
+import { createIndexingConfig } from "@ensnode/ensnode-sdk";
+import {
+  type BlockRef,
+  type BlockrangeWithStartBlock,
+  type ChainId,
+  ChainIndexingStates,
+  type PonderIndexingMetrics,
+} from "@ensnode/ponder-sdk";
+
+import type { ChainIndexingMetadataImmutable } from "@/lib/indexing-status-builder/chain-indexing-metadata";
+
+import { fetchBlockRef } from "./fetch-block-ref";
+
+/**
+ * Build an immutable indexing metadata for a chain.
+ *
+ * Some of the metadata fields are based on RPC calls to fetch block references.
+ *
+ * @param startBlock Chain's start block.
+ * @param endBlock Chain's end block (optional).
+ * @param backfillEndBlock Chain's backfill end block.
+ *
+ * @returns The immutable indexing metadata for the chain.
+ */
+function buildChainIndexingMetadataImmutable(
+  startBlock: BlockRef,
+  endBlock: BlockRef | null,
+  backfillEndBlock: BlockRef,
+): ChainIndexingMetadataImmutable {
+  const chainIndexingConfig = createIndexingConfig(startBlock, endBlock);
+
+  return {
+    backfillScope: {
+      startBlock,
+      endBlock: backfillEndBlock,
+    },
+    indexingConfig: chainIndexingConfig,
+  };
+}
+
+/**
+ * Map of chain ID to its immutable indexing metadata.
+ */
+export type ChainsIndexingMetadataImmutable = Map<ChainId, ChainIndexingMetadataImmutable>;
+
+/**
+ * Build a map of chain ID to its immutable indexing metadata.
+ *
+ * @param indexedChainIds Set of indexed chain IDs.
+ * @param chainsConfigBlockrange Map of chain ID to its configured blockrange.
+ * @param publicClients Map of chain ID to its cached public client.
+ * @param ponderIndexingMetrics Ponder indexing metrics for each chain.
+ * @returns A map of chain ID to its immutable indexing metadata.
+ * @throws Error if any of the required data cannot be fetched or is invalid,
+ *         or if invariants are violated.
+ */
+export async function buildChainsIndexingMetadataImmutable(
+  indexedChainIds: Set<ChainId>,
+  chainsConfigBlockrange: Map<ChainId, BlockrangeWithStartBlock>,
+  publicClients: Map<ChainId, PublicClient>,
+  ponderIndexingMetrics: PonderIndexingMetrics,
+): Promise<Map<ChainId, ChainIndexingMetadataImmutable>> {
+  const chainsIndexingMetadataImmutable = new Map<ChainId, ChainIndexingMetadataImmutable>();
+
+  for (const chainId of indexedChainIds) {
+    const chainConfigBlockrange = chainsConfigBlockrange.get(chainId);
+    const chainIndexingMetrics = ponderIndexingMetrics.chains.get(chainId);
+    const publicClient = publicClients.get(chainId);
+
+    // Invariants: chain config blockrange, indexing metrics, and public client
+    // must exist in proper state for the indexed chain.
+    if (!chainConfigBlockrange) {
+      throw new Error(`Chain config blockrange must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (!chainIndexingMetrics) {
+      throw new Error(`Indexing metrics must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (chainIndexingMetrics.state !== ChainIndexingStates.Historical) {
+      throw new Error(
+        `Chain indexing state must be "historical" for indexed chain ID ${chainId}, but got "${chainIndexingMetrics.state}"`,
+      );
+    }
+
+    if (!publicClient) {
+      throw new Error(`Public client must be available for indexed chain ID ${chainId}`);
+    }
+
+    const backfillEndBlockNumber =
+      chainConfigBlockrange.startBlock + chainIndexingMetrics.historicalTotalBlocks - 1;
+
+    // Fetch required block references in parallel.
+    const [startBlock, endBlock, backfillEndBlock] = await Promise.all([
+      fetchBlockRef(publicClient, chainConfigBlockrange.startBlock),
+      chainConfigBlockrange.endBlock
+        ? fetchBlockRef(publicClient, chainConfigBlockrange.endBlock)
+        : null,
+      fetchBlockRef(publicClient, backfillEndBlockNumber),
+    ]);
+
+    const metadataImmutable = buildChainIndexingMetadataImmutable(
+      startBlock,
+      endBlock,
+      backfillEndBlock,
+    );
+
+    chainsIndexingMetadataImmutable.set(chainId, metadataImmutable);
+  }
+
+  return chainsIndexingMetadataImmutable;
+}