A bunch of improvements around extracting AI data

New model admin dashboard, test model strings, add and edit models, view
missing models and easily add them. Also extract cost data from ai
gateway provider response metadata, better enrichment.

Author: ericallam

apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx

@@ -6,6 +6,9 @@ export function AITagsRow({ aiData }: { aiData: AISpanData }) {
     <div className="flex flex-wrap items-center gap-1.5 py-2.5">
       <Pill>{aiData.model}</Pill>
       {aiData.provider !== "unknown" && <Pill variant="dimmed">{aiData.provider}</Pill>}
+      {aiData.resolvedProvider && (
+        <Pill variant="dimmed">via {aiData.resolvedProvider}</Pill>
+      )}
       {aiData.finishReason && <Pill variant="dimmed">{aiData.finishReason}</Pill>}
       {aiData.serviceTier && <Pill variant="dimmed">tier: {aiData.serviceTier}</Pill>}
       {aiData.toolChoice && <Pill variant="dimmed">tools: {aiData.toolChoice}</Pill>}
@@ -38,7 +41,14 @@ export function AIStatsSummary({ aiData }: { aiData: AISpanData }) {
         <MetricRow label="Input" value={aiData.inputTokens.toLocaleString()} unit="tokens" />
         <MetricRow label="Output" value={aiData.outputTokens.toLocaleString()} unit="tokens" />
         {aiData.cachedTokens != null && aiData.cachedTokens > 0 && (
-          <MetricRow label="Cached" value={aiData.cachedTokens.toLocaleString()} unit="tokens" />
+          <MetricRow label="Cache read" value={aiData.cachedTokens.toLocaleString()} unit="tokens" />
+        )}
+        {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && (
+          <MetricRow
+            label="Cache write"
+            value={aiData.cacheCreationTokens.toLocaleString()}
+            unit="tokens"
+          />
         )}
         {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && (
           <MetricRow

apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts

@@ -34,9 +34,12 @@ export function extractAISpanData(
   const model = str(gResponse.model) ?? str(gRequest.model) ?? str(aiModel.id);
   if (!model) return undefined;
 
-  // Prefer ai.usage (richer) over gen_ai.usage
-  const inputTokens = num(aiUsage.inputTokens) ?? num(gUsage.input_tokens) ?? 0;
-  const outputTokens = num(aiUsage.outputTokens) ?? num(gUsage.output_tokens) ?? 0;
+  // Prefer ai.usage (richer) over gen_ai.usage.
+  // Gateway/some providers emit promptTokens/completionTokens instead of inputTokens/outputTokens.
+  const inputTokens =
+    num(aiUsage.inputTokens) ?? num(aiUsage.promptTokens) ?? num(gUsage.input_tokens) ?? 0;
+  const outputTokens =
+    num(aiUsage.outputTokens) ?? num(aiUsage.completionTokens) ?? num(gUsage.output_tokens) ?? 0;
   const totalTokens = num(aiUsage.totalTokens) ?? inputTokens + outputTokens;
 
   const tokensPerSecond =
@@ -56,6 +59,7 @@ export function extractAISpanData(
     operationName: str(gOperation.name) ?? str(ai.operationId) ?? "",
     finishReason: str(aiResponse.finishReason),
     serviceTier: providerMeta?.serviceTier,
+    resolvedProvider: providerMeta?.resolvedProvider,
     toolChoice: parseToolChoice(aiPrompt.toolChoice),
     toolCount: toolDefs?.length,
     messageCount: countMessages(aiPrompt.messages),
@@ -64,14 +68,16 @@ export function extractAISpanData(
     outputTokens,
     totalTokens,
     cachedTokens: num(aiUsage.cachedInputTokens) ?? num(gUsage.cache_read_input_tokens),
+    cacheCreationTokens:
+      num(aiUsage.cacheCreationInputTokens) ?? num(gUsage.cache_creation_input_tokens),
     reasoningTokens: num(aiUsage.reasoningTokens) ?? num(gUsage.reasoning_tokens),
     tokensPerSecond,
     msToFirstChunk: num(aiResponse.msToFirstChunk),
     durationMs,
     inputCost: num(triggerLlm.input_cost),
     outputCost: num(triggerLlm.output_cost),
     totalCost: num(triggerLlm.total_cost),
-    responseText: str(aiResponse.text) || undefined,
+    responseText: str(aiResponse.text) || str(aiResponse.object) || undefined,
     toolDefinitions: toolDefs,
     items: buildDisplayItems(aiPrompt.messages, aiResponse.toolCalls, toolDefs),
   };
@@ -417,18 +423,38 @@ function parseToolDefinitions(raw: unknown): ToolDefinition[] | undefined {
 
 function parseProviderMetadata(
   raw: unknown
-): { serviceTier?: string } | undefined {
+): { serviceTier?: string; resolvedProvider?: string; gatewayCost?: string } | undefined {
   if (typeof raw !== "string") return undefined;
   try {
     const parsed = JSON.parse(raw);
     if (!parsed || typeof parsed !== "object") return undefined;
 
+    let serviceTier: string | undefined;
+    let resolvedProvider: string | undefined;
+    let gatewayCost: string | undefined;
+
     // Anthropic: { anthropic: { usage: { service_tier: "standard" } } }
-    const anthropic = rec(parsed.anthropic ?? parsed);
-    const usage = rec(anthropic.usage);
-    const serviceTier = str(usage.service_tier);
+    const anthropic = rec(parsed.anthropic);
+    serviceTier = str(rec(anthropic.usage).service_tier);
+
+    // Azure/OpenAI: { azure: { serviceTier: "default" } } or { openai: { serviceTier: "..." } }
+    if (!serviceTier) {
+      serviceTier = str(rec(parsed.azure).serviceTier) ?? str(rec(parsed.openai).serviceTier);
+    }
+
+    // Gateway: { gateway: { routing: { finalProvider, resolvedProvider }, cost } }
+    const gateway = rec(parsed.gateway);
+    const routing = rec(gateway.routing);
+    resolvedProvider = str(routing.finalProvider) ?? str(routing.resolvedProvider);
+    gatewayCost = str(gateway.cost);
+
+    // OpenRouter: { openrouter: { provider: "xAI" } }
+    if (!resolvedProvider) {
+      resolvedProvider = str(rec(parsed.openrouter).provider);
+    }
 
-    return serviceTier ? { serviceTier } : undefined;
+    if (!serviceTier && !resolvedProvider && !gatewayCost) return undefined;
+    return { serviceTier, resolvedProvider, gatewayCost };
   } catch {
     return undefined;
   }

apps/webapp/app/components/runs/v3/ai/types.ts

@@ -66,6 +66,8 @@ export type AISpanData = {
   // Categorical tags
   finishReason?: string;
   serviceTier?: string;
+  /** Resolved downstream provider for gateway/openrouter spans (e.g. "xAI", "mistral") */
+  resolvedProvider?: string;
   toolChoice?: string;
   toolCount?: number;
   messageCount?: number;
@@ -77,6 +79,7 @@ export type AISpanData = {
   outputTokens: number;
   totalTokens: number;
   cachedTokens?: number;
+  cacheCreationTokens?: number;
   reasoningTokens?: number;
 
   // Performance

apps/webapp/app/env.server.ts

@@ -1247,6 +1247,12 @@ const EnvironmentSchema = z
     // Metric widget concurrency limits
     METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT: z.coerce.number().int().default(30),
 
+    // Admin ClickHouse URL (for admin dashboard queries like missing models)
+    ADMIN_CLICKHOUSE_URL: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.CLICKHOUSE_URL),
+
     EVENTS_CLICKHOUSE_URL: z
       .string()
       .optional()

apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts

@@ -0,0 +1,33 @@
+import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server";
+
+async function requireAdmin(request: Request) {
+  const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+  if (!authResult) {
+    throw json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+  if (!user?.admin) {
+    throw json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  return user;
+}
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  await requireAdmin(request);
+
+  const url = new URL(request.url);
+  const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10);
+
+  if (isNaN(lookbackHours) || lookbackHours < 1 || lookbackHours > 720) {
+    return json({ error: "lookbackHours must be between 1 and 720" }, { status: 400 });
+  }
+
+  const models = await getMissingLlmModels({ lookbackHours });
+
+  return json({ models, lookbackHours });
+}