feat(webapp): link Sentry events to OTel traces via trace_id (#3531)

## Summary

Stamps the active OpenTelemetry `trace_id` and `span_id` onto every
Sentry event captured from the webapp, so engineers can copy a
`trace_id` from a Sentry issue and search for the corresponding trace in
any OTel-aware backend. Also adds an `otel_sampled` tag to indicate
whether the trace was head-sampled — a cheap signal for whether the link
will resolve to span data or hit a missing trace.

## Why

Sentry and OTel were OTel-disconnected: `apps/webapp/sentry.server.ts`
initialised Sentry with `skipOpenTelemetrySetup: true`, and no
error-capture site (`logger.server.ts`, the Remix-wrapped `handleError`,
the root `ErrorBoundary`) attached OTel context to the event. With many
spans/sec across services, getting from a Sentry issue to its trace was
guesswork.

## Approach

Single global Sentry event processor, registered immediately after
`Sentry.init`. On each event it reads
`trace.getActiveSpan()?.spanContext()` via `@opentelemetry/api`, then
writes:

- `event.contexts.trace.trace_id` and `event.contexts.trace.span_id`
(Sentry's native trace context fields)
- `event.tags.otel_sampled` = `"true"` | `"false"` (derived from
`traceFlags`)

If no active span (module-load errors, scheduled timers without a
context, primary cluster process), the processor returns the event
unmodified — Sentry's default propagation context fills in.

Implementation is co-located in `apps/webapp/sentry.server.ts` (no
separate helper module — `sentry.server.ts` is built standalone by
esbuild and a separate import would have required a new bundling step).
Helper functions are exported so the unit tests can reach them without
re-running `Sentry.init`.

## Non-goals (deliberate)

- No sample rate change. ~95% of Sentry events will carry a `trace_id`
that returns no spans in the tracing backend (head-sampled out). The
`otel_sampled` tag makes that obvious at a glance. Raising find-rate is
a separate conversation with cost trade-offs.
- No user/org tags or `Sentry.setUser` (would need auth-helper +
per-request scope wiring across multiple worker entrypoints — separate
ticket).
- Webapp image only. No changes to supervisor or CLI workers.

## Test plan

- [x] Unit tests in `apps/webapp/test/sentryTraceContext.server.test.ts`
— 9 tests covering: helper returns \`undefined\` with no active span;
returns \`traceId\`/\`spanId\`/\`sampled=true\` for a recording span;
returns \`sampled=false\` for a non-recording span; processor leaves the
event unchanged with no active span; processor stamps
\`trace_id\`/\`span_id\` onto \`contexts.trace\`; preserves existing
\`contexts.trace\` fields; tags \`otel_sampled\` correctly for both
sampled and non-sampled cases; never throws if \`@opentelemetry/api\`
access throws.
- [x] \`pnpm run typecheck --filter webapp\` passes.
- [x] Manually verified end-to-end against a sandboxed Sentry project:
confirmed both sampled and non-sampled traces correctly populate
\`contexts.trace.trace_id\` matching the OTel ids logged from the
loader, and the \`otel_sampled\` tag appears with the expected value.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: d-cs

.server-changes/sentry-trace-id-context.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Stamp the active OpenTelemetry trace_id and span_id onto every Sentry event so issues can be cross-referenced with traces in any OTel backend.

apps/webapp/app/utils/sentryTraceContext.server.ts

@@ -0,0 +1,52 @@
+import { type Span, TraceFlags, trace } from "@opentelemetry/api";
+import type { Event, EventHint } from "@sentry/remix";
+
+export type GetActiveSpan = () => Span | undefined;
+
+const defaultGetActiveSpan: GetActiveSpan = () => trace.getActiveSpan();
+
+export function getActiveTraceIds(
+  getActiveSpan: GetActiveSpan = defaultGetActiveSpan
+): { traceId: string; spanId: string; sampled: boolean } | undefined {
+  try {
+    const span = getActiveSpan();
+    if (!span) return undefined;
+    const ctx = span.spanContext();
+    return {
+      traceId: ctx.traceId,
+      spanId: ctx.spanId,
+      sampled: (ctx.traceFlags & TraceFlags.SAMPLED) !== 0,
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+export function addOtelTraceContextToEvent(
+  event: Event,
+  _hint: EventHint,
+  getActiveSpan: GetActiveSpan = defaultGetActiveSpan
+): Event {
+  const ids = getActiveTraceIds(getActiveSpan);
+  if (!ids) return event;
+  // We intentionally overwrite Sentry's own trace_id/span_id on contexts.trace.
+  // With skipOpenTelemetrySetup: true, Sentry generates an internal trace_id
+  // unrelated to OTel; replacing it with the active OTel ids is the whole
+  // point of this processor — it makes Sentry issues navigable to the
+  // corresponding OTel trace in any backend.
+  return {
+    ...event,
+    contexts: {
+      ...event.contexts,
+      trace: {
+        ...event.contexts?.trace,
+        trace_id: ids.traceId,
+        span_id: ids.spanId,
+      },
+    },
+    tags: {
+      ...event.tags,
+      otel_sampled: ids.sampled ? "true" : "false",
+    },
+  };
+}

apps/webapp/package.json

@@ -7,7 +7,7 @@
     "build": "run-s build:** && pnpm run upload:sourcemaps",
     "build:remix": "remix build --sourcemap",
     "build:server": "esbuild --platform=node --format=cjs ./server.ts --outdir=build --sourcemap",
-    "build:sentry": "esbuild --platform=node --format=cjs ./sentry.server.ts --outdir=build --sourcemap",
+    "build:sentry": "esbuild --platform=node --format=cjs --outbase=. ./sentry.server.ts ./app/utils/sentryTraceContext.server.ts --outdir=build --sourcemap",
     "dev": "cross-env PORT=3030 remix dev -c \"node ./build/server.js\"",
     "dev:worker": "cross-env NODE_PATH=../../node_modules/.pnpm/node_modules node ./build/server.js",
     "format": "prettier --write .",

apps/webapp/sentry.server.ts

@@ -1,4 +1,5 @@
 import * as Sentry from "@sentry/remix";
+import { addOtelTraceContextToEvent } from "./app/utils/sentryTraceContext.server";
 
 if (process.env.SENTRY_DSN) {
   console.log("🔭 Initializing Sentry");
@@ -29,4 +30,6 @@ if (process.env.SENTRY_DSN) {
     ignoreErrors: ["queryRoute() call aborted", /^ServiceValidationError(?::|$)/],
     includeLocalVariables: false,
   });
+
+  Sentry.addEventProcessor(addOtelTraceContextToEvent);
 }

apps/webapp/test/sentryTraceContext.server.test.ts

@@ -0,0 +1,127 @@
+import { ROOT_CONTEXT, TraceFlags, context, trace } from "@opentelemetry/api";
+import { describe, expect, it } from "vitest";
+import {
+  addOtelTraceContextToEvent,
+  getActiveTraceIds,
+} from "../app/utils/sentryTraceContext.server";
+import { createInMemoryTracing } from "./utils/tracing";
+
+describe("getActiveTraceIds", () => {
+  it("returns undefined when no OTel span is active", () => {
+    expect(getActiveTraceIds()).toBeUndefined();
+  });
+
+  it("returns the trace_id, span_id, and sampled=true for an active recording span", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const ids = getActiveTraceIds();
+      expect(ids).toEqual({
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        sampled: true,
+      });
+      span.end();
+    });
+  });
+
+  it("returns sampled=false when the active span is non-recording", () => {
+    // Initialise the global context manager (createInMemoryTracing does this
+    // as a side effect of NodeTracerProvider.register()).
+    createInMemoryTracing();
+
+    const nonSampledSpan = trace.wrapSpanContext({
+      traceId: "0123456789abcdef0123456789abcdef",
+      spanId: "0123456789abcdef",
+      traceFlags: TraceFlags.NONE,
+    });
+
+    context.with(trace.setSpan(ROOT_CONTEXT, nonSampledSpan), () => {
+      expect(getActiveTraceIds()).toEqual({
+        traceId: "0123456789abcdef0123456789abcdef",
+        spanId: "0123456789abcdef",
+        sampled: false,
+      });
+    });
+  });
+});
+
+describe("addOtelTraceContextToEvent", () => {
+  it("returns the event unchanged when no OTel span is active", () => {
+    const event = { message: "boom" };
+    const result = addOtelTraceContextToEvent(event, {});
+    expect(result).toBe(event);
+    expect(result).toEqual({ message: "boom" });
+  });
+
+  it("stamps trace_id and span_id from the active span onto event.contexts.trace", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.contexts?.trace?.trace_id).toBe(span.spanContext().traceId);
+      expect(result.contexts?.trace?.span_id).toBe(span.spanContext().spanId);
+      span.end();
+    });
+  });
+
+  it("tags the event with otel_sampled=true when the active span is recording", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.tags?.otel_sampled).toBe("true");
+      span.end();
+    });
+  });
+
+  it("tags the event with otel_sampled=false when the active span is non-recording", () => {
+    createInMemoryTracing();
+
+    const nonSampledSpan = trace.wrapSpanContext({
+      traceId: "0123456789abcdef0123456789abcdef",
+      spanId: "0123456789abcdef",
+      traceFlags: TraceFlags.NONE,
+    });
+
+    context.with(trace.setSpan(ROOT_CONTEXT, nonSampledSpan), () => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.tags?.otel_sampled).toBe("false");
+    });
+  });
+
+  it("preserves existing event.contexts.trace fields", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = {
+        message: "boom",
+        contexts: {
+          trace: { op: "http.server", description: "GET /things" },
+          runtime: { name: "node" },
+        },
+      };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.contexts?.trace).toMatchObject({
+        op: "http.server",
+        description: "GET /things",
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+      });
+      expect(result.contexts?.runtime).toEqual({ name: "node" });
+      span.end();
+    });
+  });
+
+  it("returns the event unchanged if reading the OTel context throws", () => {
+    const throwingAccessor = () => {
+      throw new Error("otel api blew up");
+    };
+    const event = { message: "boom" };
+    const result = addOtelTraceContextToEvent(event, {}, throwingAccessor);
+    expect(result).toBe(event);
+  });
+});