fix: scope external schema compliance bundles

.changeset/scoped-schema-compliance-bundles.md

@@ -0,0 +1,5 @@
+---
+'@adcp/sdk': patch
+---
+
+Fix external compliance/schema bundle handling by scoping schema roots per run, preloading async response refs for request validators, preserving hosted stable-line aliases on the wire, and exposing schema-root options through conformance fuzzing.

bin/adcp-fuzz.js

@@ -22,6 +22,10 @@ Options:
   --list-tools                Print every tool name + its tier and exit
   --turn-budget <int>         Iterations per tool (default: 50)
   --protocol <mcp|a2a>        Transport (default: mcp)
+  --schema-version <version>  AdCP schema/cache version to validate against
+                              (alias: --compliance-version)
+  --schema-root <path>        External schema-data root for validation
+                              (alias: --validator-source)
   --auth-token <token>        Bearer token. Also reads ADCP_AUTH_TOKEN env var.
   --auth-token-cross-tenant <token>
                               Second auth token for the uniform-error paired
@@ -146,6 +150,16 @@ async function handleFuzzCommand(argv) {
         i++;
         break;
       }
+      case '--schema-version':
+      case '--compliance-version':
+        options.version = requireValue(i, a);
+        i++;
+        break;
+      case '--schema-root':
+      case '--validator-source':
+        options.schemaRoot = requireValue(i, a);
+        i++;
+        break;
       case '--auth-token':
         options.authToken = requireValue(i, '--auth-token');
         i++;
@@ -350,6 +364,8 @@ function reproduceCommand(report, failure) {
   const parts = ['adcp fuzz', quote(report.agentUrl), '--seed', String(failure.seed), '--tools', failure.tool];
   if (report.protocol && report.protocol !== 'mcp') parts.push('--protocol', report.protocol);
   if (report.turnBudget && report.turnBudget !== 50) parts.push('--turn-budget', String(report.turnBudget));
+  if (report.schemaVersion) parts.push('--schema-version', report.schemaVersion);
+  if (report.schemaRoot) parts.push('--schema-root', quote(report.schemaRoot));
   // Prefer --auto-seed over listing seeded IDs when the run used autoSeed:
   // seeded IDs are agent-generated and may differ between runs, so echoing
   // them as --fixture would mislead the user. The user should re-seed.

bin/adcp.js

@@ -325,12 +325,18 @@ function readFlagValue(args, flag) {
 function parseComplianceSelection(args) {
   const complianceVersion = readFlagValue(args, '--compliance-version');
   const complianceDir = readFlagValue(args, '--compliance-dir');
+  const schemaRoot = readFlagValue(args, '--schema-root') || readFlagValue(args, '--validator-source');
+  const hostedStableLineAlias = readFlagValue(args, '--hosted-stable-line-alias');
   return {
     complianceVersion,
     complianceDir,
+    schemaRoot,
+    hostedStableLineAlias,
     resolveOptions: {
       ...(complianceVersion && { version: complianceVersion }),
       ...(complianceDir && { complianceDir }),
+      ...(schemaRoot && { schemaRoot }),
+      ...(hostedStableLineAlias && { hostedStableLineAlias }),
     },
   };
 }
@@ -947,7 +953,7 @@ function parseAgentOptions(args) {
     storyboardsValue = args[storyboardsIndex + 1];
   }
 
-  const { complianceVersion, complianceDir } = parseComplianceSelection(args);
+  const { complianceVersion, complianceDir, schemaRoot, hostedStableLineAlias } = parseComplianceSelection(args);
 
   const platformTypeIndex = args.indexOf('--platform-type');
   let platformTypeValue = null;
@@ -1109,6 +1115,8 @@ function parseAgentOptions(args) {
     storyboardsValue,
     complianceVersion,
     complianceDir,
+    schemaRoot,
+    hostedStableLineAlias,
     platformTypeValue,
     timeoutValue,
     multiInstanceStrategyValue,
@@ -1151,6 +1159,8 @@ function parseAgentOptions(args) {
     softFail,
     complianceVersion,
     complianceDir,
+    schemaRoot,
+    hostedStableLineAlias,
   };
 }
 
@@ -1735,6 +1745,11 @@ RUN OPTIONS (full assessment):
                       storyboard resolution and wire-version defaults.
   --compliance-dir PATH
                       Use a specific compliance cache directory
+  --schema-root PATH  Use a specific schema bundle/root for validation.
+                      --validator-source is accepted as an alias.
+  --hosted-stable-line-alias VERSION
+                      Hosted badge mode: allow a stable line (e.g. 3.1)
+                      to resolve against a prerelease compliance cache.
   --file PATH         Run an ad-hoc storyboard YAML (spec evolution)
   --timeout SECONDS   Timeout in seconds (default: 120)
   --brief TEXT        Custom brief for product discovery
@@ -2164,7 +2179,8 @@ async function handleStoryboardShow(args) {
     PROTOCOL_TO_PATH,
   } = await import('../dist/lib/testing/storyboard/index.js');
   const jsonOutput = args.includes('--json');
-  const { complianceVersion, complianceDir, resolveOptions } = parseComplianceSelection(args);
+  const { complianceVersion, complianceDir, schemaRoot, hostedStableLineAlias, resolveOptions } =
+    parseComplianceSelection(args);
 
   // --specialism <slug>: resolve which storyboards are graded for a given specialism claim.
   const specialismIdx = args.indexOf('--specialism');
@@ -2181,6 +2197,8 @@ async function handleStoryboardShow(args) {
       !a.startsWith('--') &&
       a !== complianceVersion &&
       a !== complianceDir &&
+      a !== schemaRoot &&
+      a !== hostedStableLineAlias &&
       (specialismSlug === null || i !== specialismIdx + 1)
   );
   const storyboardId = positionalArgs[0];
@@ -2453,14 +2471,28 @@ async function handleStoryboardRun(args) {
     return;
   }
 
-  const { loadStoryboardFile, runStoryboard } = await import('../dist/lib/testing/storyboard/index.js');
+  const { loadStoryboardFile, runStoryboard, loadComplianceIndex, getExternalSchemaRootForCompliance } =
+    await import('../dist/lib/testing/storyboard/index.js');
   let storyboard;
   try {
     storyboard = loadStoryboardFile(filePath);
   } catch (err) {
     console.error(`Failed to load storyboard from ${filePath}: ${err.message}`);
     process.exit(2);
   }
+  let fileComplianceVersion = opts.complianceVersion;
+  let fileSchemaRoot = opts.schemaRoot;
+  if (opts.complianceDir || opts.complianceVersion || opts.schemaRoot) {
+    try {
+      const resolveOptions = parseComplianceSelection(args).resolveOptions;
+      const index = loadComplianceIndex(resolveOptions);
+      fileComplianceVersion = fileComplianceVersion || index.adcp_version;
+      fileSchemaRoot = fileSchemaRoot || getExternalSchemaRootForCompliance(resolveOptions, index.adcp_version);
+    } catch (err) {
+      console.error(`ERROR: ${err.message}`);
+      process.exit(2);
+    }
+  }
 
   const {
     agentUrl,
@@ -2553,6 +2585,8 @@ async function handleStoryboardRun(args) {
       resolvedOauthClientCredentials,
     }),
     ...(webhookReceiverOpts ?? {}),
+    ...(fileComplianceVersion && { adcpVersion: fileComplianceVersion }),
+    ...(fileSchemaRoot && { schemaRoot: fileSchemaRoot }),
     ...(opts.noSandbox && { sandbox: false, disable_sandbox: true }),
     ...(opts.assertsSeededState && { assertsSeededState: true }),
     ...(mergedRunHeaders && { headers: mergedRunHeaders }),
@@ -3388,10 +3422,11 @@ async function handleLocalAgentStoryboardRun(modulePath, args, opts) {
       createAgent,
       storyboards: storyboardsSpec,
       compliance: resolveOptions,
-      ...(opts.complianceVersion || opts.noSandbox || opts.assertsSeededState
+      ...(opts.complianceVersion || opts.schemaRoot || opts.noSandbox || opts.assertsSeededState
         ? {
             runStoryboardOptions: {
               ...(opts.complianceVersion && !opts.complianceDir && { adcpVersion: opts.complianceVersion }),
+              ...(opts.schemaRoot && { schemaRoot: opts.schemaRoot }),
               ...(opts.noSandbox && { sandbox: false, disable_sandbox: true }),
               ...(opts.assertsSeededState && { assertsSeededState: true }),
             },
@@ -3781,6 +3816,7 @@ async function handleMultiInstanceStoryboardRun(args, opts, urls) {
     multi_instance_strategy: strategy,
     ...(webhookReceiverOpts ?? {}),
     ...(opts.complianceVersion && !opts.complianceDir && { adcpVersion: opts.complianceVersion }),
+    ...(opts.schemaRoot && { schemaRoot: opts.schemaRoot }),
     ...(opts.noSandbox && { sandbox: false, disable_sandbox: true }),
     ...(opts.assertsSeededState && { assertsSeededState: true }),
   };
@@ -4042,6 +4078,7 @@ async function handleAgentsRoutedStoryboardRun(args, opts, routing) {
     ...(routing.default_agent ? { default_agent: routing.default_agent } : {}),
     ...(webhookReceiverOpts ?? {}),
     ...(opts.complianceVersion && !opts.complianceDir && { adcpVersion: opts.complianceVersion }),
+    ...(opts.schemaRoot && { schemaRoot: opts.schemaRoot }),
     ...(opts.noSandbox && { sandbox: false, disable_sandbox: true }),
     ...(opts.assertsSeededState && { assertsSeededState: true }),
   };
@@ -4260,6 +4297,8 @@ async function runFullAssessment(agentArg, rawArgs, parsedOpts) {
     ...(loadedTestKit && { test_kit: loadedTestKit }),
     ...(opts.complianceVersion && { version: opts.complianceVersion }),
     ...(opts.complianceDir && { complianceDir: opts.complianceDir }),
+    ...(opts.schemaRoot && { schemaRoot: opts.schemaRoot }),
+    ...(opts.hostedStableLineAlias && { hostedStableLineAlias: opts.hostedStableLineAlias }),
   };
 
   if (!opts.jsonOutput) {
@@ -4399,7 +4438,8 @@ async function runFullAssessment(agentArg, rawArgs, parsedOpts) {
 
 async function handleStoryboardStepCmd(args) {
   const { getComplianceStoryboardById, runStoryboardStep } = await import('../dist/lib/testing/storyboard/index.js');
-  const { authToken, authScheme, protocolFlag, jsonOutput, debug, positionalArgs } = parseAgentOptions(args);
+  const { authToken, authScheme, protocolFlag, jsonOutput, positionalArgs, complianceVersion, schemaRoot } =
+    parseAgentOptions(args);
   const { resolveOptions } = parseComplianceSelection(args);
 
   enforceStrictFlags(args, warnRemovedFlags(args));
@@ -4447,6 +4487,8 @@ async function handleStoryboardStepCmd(args) {
     protocol,
     context,
     request,
+    ...(complianceVersion && { adcpVersion: complianceVersion }),
+    ...(schemaRoot && { schemaRoot }),
     ...buildResolvedAuthOption({
       resolvedAuth,
       resolvedAuthScheme,

src/lib/conformance/index.ts

@@ -22,6 +22,7 @@
 export { runConformance } from './runConformance';
 export { seedFixtures } from './seeder';
 export type { SeedOptions, SeedResult, SeederName, SeedWarning } from './seeder';
+export type { ConformanceSchemaOptions } from './schemaLoader';
 export {
   STATELESS_TIER_TOOLS,
   REFERENTIAL_STATELESS_TOOLS,

src/lib/conformance/oracle.ts

@@ -2,7 +2,7 @@ import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import type { TaskResult } from '../core/ConversationTypes';
 import type { ConformanceToolName, OracleVerdict } from './types';
-import { loadResponseSchema } from './schemaLoader';
+import { loadResponseSchema, type ConformanceSchemaOptions } from './schemaLoader';
 
 export interface OracleInput {
   tool: ConformanceToolName;
@@ -11,6 +11,8 @@ export interface OracleInput {
   result: TaskResult<unknown>;
   /** Auth token that was used to make the request — checked for leaks. */
   authToken?: string;
+  /** Schema bundle override for same-PR / external conformance runs. */
+  schemas?: ConformanceSchemaOptions;
 }
 
 export interface OracleOutput {
@@ -62,13 +64,19 @@ function getAjv(): Ajv {
   return ajv;
 }
 
-const compiledValidators = new Map<ConformanceToolName, ReturnType<Ajv['compile']>>();
-function responseValidator(tool: ConformanceToolName): ReturnType<Ajv['compile']> {
-  const cached = compiledValidators.get(tool);
+const compiledValidators = new Map<string, ReturnType<Ajv['compile']>>();
+
+function schemaCacheKey(tool: ConformanceToolName, options: ConformanceSchemaOptions | undefined): string {
+  return `${tool}\0${options?.schemaRoot ?? ''}\0${options?.version ?? ''}`;
+}
+
+function responseValidator(tool: ConformanceToolName, options?: ConformanceSchemaOptions): ReturnType<Ajv['compile']> {
+  const cacheKey = schemaCacheKey(tool, options);
+  const cached = compiledValidators.get(cacheKey);
   if (cached) return cached;
-  const schema = loadResponseSchema(tool);
+  const schema = loadResponseSchema(tool, options);
   const validator = getAjv().compile(schema);
-  compiledValidators.set(tool, validator);
+  compiledValidators.set(cacheKey, validator);
   return validator;
 }
 
@@ -78,18 +86,19 @@ function responseValidator(tool: ConformanceToolName): ReturnType<Ajv['compile']
  * request-context that's not echoed is an invariant violation; when it
  * doesn't, a missing context field is silent tolerance.
  */
-const responseSchemaHasContext = new Map<ConformanceToolName, boolean>();
-function responseEchoesContext(tool: ConformanceToolName): boolean {
-  const cached = responseSchemaHasContext.get(tool);
+const responseSchemaHasContext = new Map<string, boolean>();
+function responseEchoesContext(tool: ConformanceToolName, options?: ConformanceSchemaOptions): boolean {
+  const cacheKey = schemaCacheKey(tool, options);
+  const cached = responseSchemaHasContext.get(cacheKey);
   if (cached !== undefined) return cached;
-  const schema = loadResponseSchema(tool) as {
+  const schema = loadResponseSchema(tool, options) as {
     properties?: Record<string, unknown>;
     oneOf?: Array<{ properties?: Record<string, unknown> }>;
   };
   const direct = !!schema.properties && 'context' in schema.properties;
   const branched = Array.isArray(schema.oneOf) && schema.oneOf.some(b => !!b.properties && 'context' in b.properties);
   const answer = direct || branched;
-  responseSchemaHasContext.set(tool, answer);
+  responseSchemaHasContext.set(cacheKey, answer);
   return answer;
 }
 
@@ -99,8 +108,8 @@ function responseEchoesContext(tool: ConformanceToolName): boolean {
  * than fail every iteration. Separated from `evaluate()` so the failure
  * surfaces once per tool, not once per sample.
  */
-export function prepareResponseValidator(tool: ConformanceToolName): void {
-  responseValidator(tool);
+export function prepareResponseValidator(tool: ConformanceToolName, options?: ConformanceSchemaOptions): void {
+  responseValidator(tool, options);
 }
 
 /**
@@ -117,13 +126,13 @@ export function prepareResponseValidator(tool: ConformanceToolName): void {
  * echoed unchanged when the caller supplied one.
  */
 export function evaluate(input: OracleInput): OracleOutput {
-  const { tool, request, result, authToken } = input;
+  const { tool, request, result, authToken, schemas } = input;
   const invariantFailures: string[] = [];
 
   checkNoAuthLeak(result, authToken, invariantFailures);
   checkNoStackLeak(result, invariantFailures);
   checkNoFilesystemLeak(result, invariantFailures);
-  checkContextEchoed(tool, request, result, invariantFailures);
+  checkContextEchoed(tool, request, result, invariantFailures, schemas);
 
   if (result.success === false) {
     checkErrorEnvelope(result, invariantFailures);
@@ -139,7 +148,7 @@ export function evaluate(input: OracleInput): OracleOutput {
     return { verdict: 'rejected', invariantFailures };
   }
 
-  const validate = responseValidator(tool);
+  const validate = responseValidator(tool, schemas);
   const payloadForValidation = responsePayloadForValidation(result);
   if (!validate(payloadForValidation)) {
     const errors = (validate.errors ?? []).slice(0, 3).map(formatAjvError);
@@ -215,7 +224,8 @@ function checkContextEchoed(
   tool: ConformanceToolName,
   request: unknown,
   result: TaskResult<unknown>,
-  failures: string[]
+  failures: string[],
+  schemas?: ConformanceSchemaOptions
 ): void {
   const reqContext = (request as { context?: unknown } | undefined)?.context;
   if (reqContext === undefined) return;
@@ -225,7 +235,7 @@ function checkContextEchoed(
     // echo IS a violation — the spec requires unchanged pass-through.
     // When the schema omits `context` (some discovery-only responses do),
     // silent tolerance stands.
-    if (responseEchoesContext(tool)) {
+    if (responseEchoesContext(tool, schemas)) {
       failures.push('request.context not echoed on response (schema declares context but response omitted it)');
     }
     return;