realm-server: thread indexing job_id through prerender for log correlation

packages/realm-server/middleware/index.ts

@@ -13,6 +13,10 @@ import {
   AuthenticationErrorMessages,
   SupportedMimeType,
 } from '@cardstack/runtime-common/router';
+import {
+  PRERENDER_JOB_ID_HEADER,
+  sanitizePrerenderJobId,
+} from '../prerender/prerender-constants';
 
 const REQUEST_BODY_STATE = 'requestBody';
 
@@ -80,18 +84,24 @@ export function healthCheck(ctxt: Koa.Context, next: Koa.Next) {
 
 export function httpLogging(ctxt: Koa.Context, next: Koa.Next) {
   let logger = getLogger('realm:requests');
+  // Stamp `[job: J.R]` onto the request log lines when the upstream
+  // caller (typically the worker indexing pipeline) supplied an
+  // `x-boxel-job-id` header. Lets a single substring filter pull
+  // realm-server lines for an indexing job alongside worker lines.
+  let jobId = sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER));
+  let jobTag = jobId ? ` [job: ${jobId}]` : '';
 
   logger.info(
     `<-- ${ctxt.method} ${ctxt.req.headers.accept} ${
       fullRequestURL(ctxt).href
-    }`,
+    }${jobTag}`,
   );
 
   ctxt.res.on('finish', () => {
     logger.info(
       `--> ${ctxt.method} ${ctxt.req.headers.accept} ${
         fullRequestURL(ctxt).href
-      }: ${ctxt.status}`,
+      }: ${ctxt.status}${jobTag}`,
     );
     logger.debug(JSON.stringify(ctxt.req.headers));
   });

packages/realm-server/prerender/manager-app.ts

@@ -4,11 +4,13 @@ import { logger } from '@cardstack/runtime-common';
 import { fetchRequestFromContext, fullRequestURL } from '../middleware';
 import { format } from 'date-fns';
 import {
+  PRERENDER_JOB_ID_HEADER,
   PRERENDER_REQUEST_ID_HEADER,
   PRERENDER_SERVER_DRAINING_STATUS_CODE,
   PRERENDER_SERVER_STATUS_DRAINING,
   PRERENDER_SERVER_STATUS_HEADER,
   resolvePrerenderServerProxyTimeoutMs,
+  sanitizePrerenderJobId,
   sanitizePrerenderRequestId,
 } from './prerender-constants';
 import { randomUUID } from 'crypto';
@@ -905,6 +907,13 @@ export function buildPrerenderManagerApp(options?: {
       sanitizePrerenderRequestId(ctxt.get(PRERENDER_REQUEST_ID_HEADER)) ??
       randomUUID();
     ctxt.set(PRERENDER_REQUEST_ID_HEADER, requestId);
+    // Optional indexing-job correlator threaded from the worker. When
+    // present, gets stamped onto every proxying/proxied log line as
+    // `[job: J.R]` and forwarded upstream so the prerender-server can
+    // do the same — letting `|= "[job: J.R]"` filters return all four
+    // services' lines for an indexing job.
+    let jobId = sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER));
+    let jobTag = jobId ? ` [job: ${jobId}]` : '';
     let proxyStart = now();
     // Propagate client-disconnect into the upstream fetch so the
     // prerender server can cancel whatever it was doing and free
@@ -1060,7 +1069,7 @@ export function buildPrerenderManagerApp(options?: {
         let logTarget = attrs.url ?? attrs.command ?? '<unknown>';
         let queueMs = now() - proxyStart;
         log.info(
-          `proxying ${label} prerender request for ${logTarget} to ${targetURL} requestId=${requestId} affinity=${affinityKey} attempt=${attempts.size} queueMs=${queueMs}`,
+          `proxying ${label} prerender request for ${logTarget} to ${targetURL} requestId=${requestId} affinity=${affinityKey} attempt=${attempts.size} queueMs=${queueMs}${jobTag}`,
         );
         let abortedDueToDrain = false;
         const ac = new AbortController();
@@ -1094,6 +1103,7 @@ export function buildPrerenderManagerApp(options?: {
             'Content-Type': 'application/vnd.api+json',
             Accept: ctxt.get('Accept') || 'application/vnd.api+json',
             [PRERENDER_REQUEST_ID_HEADER]: requestId,
+            ...(jobId ? { [PRERENDER_JOB_ID_HEADER]: jobId } : {}),
           },
           body: raw,
           signal: ac.signal,
@@ -1204,7 +1214,7 @@ export function buildPrerenderManagerApp(options?: {
         ctxt.body = buf;
         let proxyMs = now() - proxyStart;
         log.info(
-          `proxied ${label} requestId=${requestId} affinity=${affinityKey} target=${target} status=${res.status} proxyMs=${proxyMs}`,
+          `proxied ${label} requestId=${requestId} affinity=${affinityKey} target=${target} status=${res.status} proxyMs=${proxyMs}${jobTag}`,
         );
         return;
       }
@@ -1347,15 +1357,17 @@ export function buildPrerenderManagerApp(options?: {
     process.env.PRERENDER_MANAGER_VERBOSE_LOGS === 'true';
   app
     .use((ctxt: Koa.Context, next: Koa.Next) => {
+      let jobId = sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER));
+      let jobTag = jobId ? ` [job: ${jobId}]` : '';
       if (verboseManagerLogs) {
         log.info(
-          `<-- ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}`,
+          `<-- ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}${jobTag}`,
         );
       }
       ctxt.res.on('finish', () => {
         if (verboseManagerLogs) {
           log.info(
-            `--> ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}: ${ctxt.status}`,
+            `--> ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}: ${ctxt.status}${jobTag}`,
           );
           log.debug(JSON.stringify(ctxt.req.headers));
         }

packages/realm-server/prerender/prerender-app.ts

@@ -22,10 +22,12 @@ import { Prerenderer } from './index';
 import type { Timings } from './render-runner';
 import { resolvePrerenderManagerURL } from './config';
 import {
+  PRERENDER_JOB_ID_HEADER,
   PRERENDER_REQUEST_ID_HEADER,
   PRERENDER_SERVER_DRAINING_STATUS_CODE,
   PRERENDER_SERVER_STATUS_DRAINING,
   PRERENDER_SERVER_STATUS_HEADER,
+  sanitizePrerenderJobId,
   sanitizePrerenderRequestId,
 } from './prerender-constants';
 import { randomUUID } from 'crypto';
@@ -957,12 +959,14 @@ export function buildPrerenderApp(options: {
       return next();
     })
     .use((ctxt: Koa.Context, next: Koa.Next) => {
+      let jobId = sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER));
+      let jobTag = jobId ? ` [job: ${jobId}]` : '';
       log.info(
-        `<-- ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}`,
+        `<-- ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}${jobTag}`,
       );
       ctxt.res.on('finish', () => {
         log.info(
-          `--> ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}: ${ctxt.status}`,
+          `--> ${ctxt.method} ${ctxt.req.headers.accept} ${fullRequestURL(ctxt).href}: ${ctxt.status}${jobTag}`,
         );
         log.debug(JSON.stringify(ctxt.req.headers));
       });

packages/realm-server/prerender/prerender-constants.ts

@@ -27,6 +27,29 @@ export function sanitizePrerenderRequestId(
   return REQUEST_ID_PATTERN.test(trimmed) ? trimmed : null;
 }
 
+// Threads the indexing job's `<jobId>.<reservationId>` identifier from
+// the worker through the prerender call chain (worker → manager →
+// prerender-server). Any service that handles the request stamps
+// `[job: J.R]` onto its `<--`/`-->` HTTP-log lines and the manager's
+// `proxying`/`proxied` lines so a single job's prerender activity is
+// greppable across services with the same `[job: J.R]` substring used
+// in worker logs.
+export const PRERENDER_JOB_ID_HEADER = 'x-boxel-job-id';
+
+// Sanitize the inbound job-id header. Format is `<digits>.<digits>`
+// (job.id + reservation.id, both bigint-shaped); accept up to 32
+// digits per side (so up to 65 chars total including the separator)
+// to be defensive without admitting newlines or other log-injection.
+const JOB_ID_PATTERN = /^[0-9]{1,32}\.[0-9]{1,32}$/;
+export function sanitizePrerenderJobId(
+  raw: string | null | undefined,
+): string | null {
+  if (typeof raw !== 'string') return null;
+  let trimmed = raw.trim();
+  if (!trimmed) return null;
+  return JOB_ID_PATTERN.test(trimmed) ? trimmed : null;
+}
+
 // Base timeout for a single prerender capture on the prerender server
 // (DOM rendering + data loading inside the headless browser).
 const DEFAULT_RENDER_TIMEOUT_MS = 90_000;

packages/realm-server/prerender/remote-prerenderer.ts

@@ -10,11 +10,13 @@ import {
   logger,
 } from '@cardstack/runtime-common';
 import {
+  PRERENDER_JOB_ID_HEADER,
   PRERENDER_REQUEST_ID_HEADER,
   PRERENDER_SERVER_DRAINING_STATUS_CODE,
   PRERENDER_SERVER_STATUS_DRAINING,
   PRERENDER_SERVER_STATUS_HEADER,
   resolvePrerenderManagerRequestTimeoutMs,
+  sanitizePrerenderJobId,
 } from './prerender-constants';
 import { randomUUID } from 'crypto';
 
@@ -70,12 +72,21 @@ export function createRemotePrerenderer(
     validatePrerenderAttributes(type, attributes);
 
     let endpoint = new URL(path, prerenderURL);
+    // jobId is request metadata, not part of the validated body — strip
+    // it out before sending so the prerender-server's payload schema
+    // doesn't need to know about it.
+    let { jobId, ...attributesWithoutJobId } = attributes as Record<
+      string,
+      any
+    >;
     let body = {
       data: {
         type,
-        attributes,
+        attributes: attributesWithoutJobId,
       },
     };
+    let sanitizedJobId =
+      typeof jobId === 'string' ? sanitizePrerenderJobId(jobId) : null;
     // CS-10872: one correlation ID per logical client call, reused on
     // retries so operators can follow the full manager/prerender-server
     // log story for the same intent rather than hunting through N
@@ -97,6 +108,9 @@ export function createRemotePrerenderer(
           headers: {
             ...jsonApiHeaders,
             [PRERENDER_REQUEST_ID_HEADER]: requestId,
+            ...(sanitizedJobId
+              ? { [PRERENDER_JOB_ID_HEADER]: sanitizedJobId }
+              : {}),
           },
           body: JSON.stringify(body),
           signal: ac.signal,
@@ -208,6 +222,7 @@ export function createRemotePrerenderer(
       types,
       batchId,
       priority,
+      jobId,
     }: PrerenderVisitArgs): Promise<RenderVisitResponse> {
       return await requestWithRetry<RenderVisitResponse>(
         'prerender-visit',
@@ -223,6 +238,7 @@ export function createRemotePrerenderer(
           ...(types ? { types } : {}),
           ...(batchId ? { batchId } : {}),
           ...(priority !== undefined ? { priority } : {}),
+          ...(jobId ? { jobId } : {}),
         },
       );
     },

packages/realm-server/tests/remote-prerenderer-test.ts

@@ -212,6 +212,128 @@ module(basename(__filename), function (hooks) {
         /Missing prerender run-command-request attributes: affinityValue/,
       );
     });
+
+    test('threads jobId through as x-boxel-job-id header and strips from body', async function (assert) {
+      let receivedHeaders: any;
+      let receivedBody: any;
+      let server = createServer((req, res) => {
+        receivedHeaders = req.headers;
+        let body: Buffer[] = [];
+        req.on('data', (chunk) => body.push(chunk));
+        req.on('end', () => {
+          receivedBody = JSON.parse(Buffer.concat(body).toString('utf-8'));
+          res.statusCode = 201;
+          res.setHeader('Content-Type', 'application/json');
+          res.end(
+            JSON.stringify({
+              data: { attributes: { ok: true } },
+            }),
+          );
+        });
+      }).listen(0);
+
+      try {
+        let url = `http://127.0.0.1:${(server.address() as any).port}`;
+        let prerenderer = createRemotePrerenderer(url);
+
+        await prerenderer.prerenderVisit({
+          affinityType: 'realm',
+          affinityValue: 'realm-1',
+          realm: 'realm-1',
+          url: 'https://example.com/card.json',
+          auth: '{}',
+          jobId: '20678.26619',
+        });
+
+        assert.strictEqual(
+          receivedHeaders?.['x-boxel-job-id'],
+          '20678.26619',
+          'jobId is sent as x-boxel-job-id header',
+        );
+        assert.notOk(
+          'jobId' in (receivedBody?.data?.attributes ?? {}),
+          'jobId is not present in data.attributes (request metadata, not payload)',
+        );
+      } finally {
+        await new Promise<void>((resolve) => server.close(() => resolve()));
+      }
+    });
+
+    test('omits x-boxel-job-id header when jobId is not provided', async function (assert) {
+      let receivedHeaders: any;
+      let server = createServer((req, res) => {
+        receivedHeaders = req.headers;
+        req.on('data', () => {});
+        req.on('end', () => {
+          res.statusCode = 201;
+          res.setHeader('Content-Type', 'application/json');
+          res.end(JSON.stringify({ data: { attributes: { ok: true } } }));
+        });
+      }).listen(0);
+
+      try {
+        let url = `http://127.0.0.1:${(server.address() as any).port}`;
+        let prerenderer = createRemotePrerenderer(url);
+
+        await prerenderer.prerenderVisit({
+          affinityType: 'realm',
+          affinityValue: 'realm-1',
+          realm: 'realm-1',
+          url: 'https://example.com/card.json',
+          auth: '{}',
+        });
+
+        assert.notOk(
+          'x-boxel-job-id' in (receivedHeaders ?? {}),
+          'x-boxel-job-id header absent when jobId is omitted',
+        );
+      } finally {
+        await new Promise<void>((resolve) => server.close(() => resolve()));
+      }
+    });
+
+    test('omits x-boxel-job-id header when jobId fails sanitization', async function (assert) {
+      let receivedHeaders: any;
+      let receivedBody: any;
+      let server = createServer((req, res) => {
+        receivedHeaders = req.headers;
+        let body: Buffer[] = [];
+        req.on('data', (chunk) => body.push(chunk));
+        req.on('end', () => {
+          receivedBody = JSON.parse(Buffer.concat(body).toString('utf-8'));
+          res.statusCode = 201;
+          res.setHeader('Content-Type', 'application/json');
+          res.end(JSON.stringify({ data: { attributes: { ok: true } } }));
+        });
+      }).listen(0);
+
+      try {
+        let url = `http://127.0.0.1:${(server.address() as any).port}`;
+        let prerenderer = createRemotePrerenderer(url);
+
+        // Newline in the job-id string is the canonical log-injection
+        // shape that sanitizePrerenderJobId is designed to reject.
+        await prerenderer.prerenderVisit({
+          affinityType: 'realm',
+          affinityValue: 'realm-1',
+          realm: 'realm-1',
+          url: 'https://example.com/card.json',
+          auth: '{}',
+          jobId: '20678.26619\nX-Injected: bad',
+        });
+
+        assert.notOk(
+          'x-boxel-job-id' in (receivedHeaders ?? {}),
+          'x-boxel-job-id header absent when jobId fails sanitization',
+        );
+        assert.notOk(
+          'jobId' in (receivedBody?.data?.attributes ?? {}),
+          'jobId stripped from body even when sanitization rejects it',
+        );
+      } finally {
+        await new Promise<void>((resolve) => server.close(() => resolve()));
+      }
+    });
   });
 
   module('remote prerenderer retries', function () {

packages/runtime-common/index-runner/visit-file.ts

@@ -173,6 +173,9 @@ export async function visitFileForIndexingFused({
       renderOptions,
       batchId,
       ...(jobPriority !== undefined ? { priority: jobPriority } : {}),
+      ...(jobInfo
+        ? { jobId: `${jobInfo.jobId}.${jobInfo.reservationId}` }
+        : {}),
     });
   } catch (err) {
     logWarn(

packages/runtime-common/index.ts

@@ -399,6 +399,14 @@ export type PrerenderVisitArgs = {
   // Worker-job priority threaded through from the producer side. See
   // ModulePrerenderArgs for the contract.
   priority?: number;
+  // `<jobId>.<reservationId>` of the indexing job that triggered this
+  // visit. Threaded through to manager + prerender-server as
+  // `x-boxel-job-id` so all three services tag their logs with
+  // `[job: J.R]` — same substring already emitted by worker code,
+  // making `{service=~"realm-server|worker|prerender|prerender-manager"}
+  // |= "[job: J.R]"` a single reliable filter for "everything that
+  // happened during this indexing job."
+  jobId?: string;
 };
 
 // Arguments for releasing an indexing batch's ownership of an affinity,