Tighten post-action effect verification

src/drivers/playwright.ts

@@ -344,13 +344,24 @@ export class PlaywrightDriver implements Driver {
     }
   }
 
+  private async captureBounds(locator: import('playwright').Locator): Promise<ActionResult['bounds']> {
+    try {
+      const box = await locator.boundingBox({ timeout: 2000 });
+      if (box) return { x: Math.round(box.x), y: Math.round(box.y), width: Math.round(box.width), height: Math.round(box.height) };
+    } catch {
+      // Element may be gone after navigation — non-critical
+    }
+    return undefined;
+  }
+
   async execute(action: Action): Promise<ActionResult> {
     const timeout = this.options.timeout ?? 30000;
 
     try {
       switch (action.action) {
         case 'click': {
           const locator = this.snapshot.resolveLocator(this.page, action.selector);
+          const bounds = await this.captureBounds(locator);
           // Listen for popups but don't block: collect any that fire during the click
           let popupPage: import('playwright').Page | null = null;
           const onPopup = (page: import('playwright').Page) => { popupPage = page; };
@@ -376,11 +387,12 @@ export class PlaywrightDriver implements Driver {
             await popupPage.waitForLoadState('domcontentloaded').catch(() => {});
             await this.adoptPage(popupPage);
           }
-          return { success: true };
+          return { success: true, bounds };
         }
 
         case 'type': {
           const locator = this.snapshot.resolveLocator(this.page, action.selector);
+          const bounds = await this.captureBounds(locator);
           try {
             await this.withOverlayRecovery(async () => {
               await locator.click({ timeout });
@@ -400,27 +412,30 @@ export class PlaywrightDriver implements Driver {
               el.dispatchEvent(new Event('change', { bubbles: true }));
             });
           }
-          return { success: true };
+          return { success: true, bounds };
         }
 
         case 'press': {
           const locator = this.snapshot.resolveLocator(this.page, action.selector);
+          const bounds = await this.captureBounds(locator);
           await this.withOverlayRecovery(async () => {
             await locator.press(action.key, { timeout });
           });
-          return { success: true };
+          return { success: true, bounds };
         }
 
         case 'hover': {
           const locator = this.snapshot.resolveLocator(this.page, action.selector);
+          const bounds = await this.captureBounds(locator);
           await locator.hover({ timeout });
-          return { success: true };
+          return { success: true, bounds };
         }
 
         case 'select': {
           const locator = this.snapshot.resolveLocator(this.page, action.selector);
+          const bounds = await this.captureBounds(locator);
           await locator.selectOption(action.value, { timeout });
-          return { success: true };
+          return { success: true, bounds };
         }
 
         case 'scroll': {

src/drivers/types.ts

@@ -4,11 +4,20 @@
 
 import type { Action, PageState } from '../types.js';
 
+export interface ActionBounds {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
 export interface ActionResult {
   success: boolean;
   error?: string;
   /** Data returned by the action (e.g., runScript result) */
   data?: string;
+  /** Bounding box of the target element at action time (for replay overlays) */
+  bounds?: ActionBounds;
 }
 
 export interface ResourceBlockingOptions {

src/runner.ts

@@ -25,6 +25,9 @@ export {
   shouldAcceptScriptBackedCompletion,
   detectCompletionContentTypeMismatch,
 
+  // Effect verification
+  verifyExpectedEffect,
+
   // Page analysis
   detectAiTangleVerifiedOutputState,
   detectAiTanglePartnerTemplateVisibleState,

src/runner/effect-verification.ts

@@ -0,0 +1,208 @@
+import type { PageState } from '../types.js';
+import { normalizeLooseText } from './utils.js';
+
+const GENERIC_EFFECT_TOKENS = new Set([
+  'a',
+  'an',
+  'and',
+  'be',
+  'become',
+  'browser',
+  'change',
+  'contain',
+  'continue',
+  'directly',
+  'effect',
+  'exact',
+  'have',
+  'include',
+  'into',
+  'it',
+  'its',
+  'list',
+  'page',
+  'query',
+  'remaining',
+  'result',
+  'results',
+  'run',
+  'search',
+  'should',
+  'site',
+  'state',
+  'switch',
+  'task',
+  'that',
+  'the',
+  'their',
+  'them',
+  'to',
+  'visible',
+]);
+
+export interface EffectVerificationInput {
+  expectedEffect: string;
+  preActionState: PageState;
+  postActionState: PageState;
+}
+
+export interface EffectVerificationResult {
+  verified: boolean;
+  reason?: string;
+}
+
+function hasStateChanged(pre: PageState, post: PageState): boolean {
+  return (
+    pre.url !== post.url
+    || pre.title !== post.title
+    || pre.snapshot !== post.snapshot
+  );
+}
+
+function getAddedLines(preSnapshot: string, postSnapshot: string): string[] {
+  const prior = new Set(
+    preSnapshot.split('\n').map((line) => line.trim()).filter(Boolean),
+  );
+  return postSnapshot
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !prior.has(line));
+}
+
+function getRemovedLines(preSnapshot: string, postSnapshot: string): string[] {
+  const next = new Set(
+    postSnapshot.split('\n').map((line) => line.trim()).filter(Boolean),
+  );
+  return preSnapshot
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !next.has(line));
+}
+
+function includesLoose(haystack: string, needle: string): boolean {
+  const normalizedHaystack = normalizeLooseText(haystack);
+  const normalizedNeedle = normalizeLooseText(needle);
+  return normalizedNeedle.length > 0 && normalizedHaystack.includes(normalizedNeedle);
+}
+
+function getCandidateKeywords(expectedEffect: string): string[] {
+  const normalized = normalizeLooseText(expectedEffect);
+  return normalized
+    .split(' ')
+    .map((token) => token.trim())
+    .filter((token) => token.length >= 4 && !GENERIC_EFFECT_TOKENS.has(token));
+}
+
+function matchesKeywordDelta(expectedEffect: string, pre: PageState, post: PageState): boolean {
+  const keywords = getCandidateKeywords(expectedEffect);
+  if (keywords.length === 0) return false;
+
+  const preText = `${pre.url}\n${pre.title}\n${pre.snapshot}`;
+  const postText = `${post.url}\n${post.title}\n${post.snapshot}`;
+  return keywords.some((keyword) => includesLoose(postText, keyword) && !includesLoose(preText, keyword));
+}
+
+export function verifyExpectedEffect({
+  expectedEffect,
+  preActionState,
+  postActionState,
+}: EffectVerificationInput): EffectVerificationResult {
+  if (/url\s+should/i.test(expectedEffect)) {
+    const currentUrl = postActionState.url;
+    const quotedVal = expectedEffect.match(/['"]([^'"]+)['"]/);
+    const verbVal = expectedEffect.match(/url\s+should\s+(?:contain|include|have)\s+(\S+)/i);
+    const expected = quotedVal?.[1] ?? verbVal?.[1];
+
+    if (expected) {
+      if (currentUrl.includes(expected)) {
+        return { verified: true };
+      }
+      return {
+        verified: false,
+        reason: `Expected URL to contain "${expected}" but got "${currentUrl}"`,
+      };
+    }
+
+    if (currentUrl !== preActionState.url) {
+      return { verified: true };
+    }
+    return {
+      verified: false,
+      reason: `Expected URL to change but it stayed at "${currentUrl}"`,
+    };
+  }
+
+  const changed = hasStateChanged(preActionState, postActionState);
+  const effectLower = expectedEffect.toLowerCase();
+  const quotedMatch = expectedEffect.match(/["']([^"']+)["']/);
+  const addedLines = getAddedLines(preActionState.snapshot, postActionState.snapshot);
+  const removedLines = getRemovedLines(preActionState.snapshot, postActionState.snapshot);
+  const postText = `${postActionState.url}\n${postActionState.title}\n${postActionState.snapshot}`;
+
+  if (quotedMatch) {
+    const searchText = quotedMatch[1];
+    if (includesLoose(postText, searchText)) {
+      return { verified: true };
+    }
+    return {
+      verified: false,
+      reason: `Expected "${searchText}" to appear in the updated page state`,
+    };
+  }
+
+  if (/(close|dismiss|hide|disappear|go away)/i.test(effectLower)) {
+    if (/\bmodal\b|\bdialog\b|\balertdialog\b/.test(effectLower)) {
+      const preHasDialog = /\b(dialog|alertdialog)\b/i.test(preActionState.snapshot);
+      const postHasDialog = /\b(dialog|alertdialog)\b/i.test(postActionState.snapshot);
+      if (preHasDialog && !postHasDialog) {
+        return { verified: true };
+      }
+    }
+    if (removedLines.length > 0 && changed) {
+      return { verified: true };
+    }
+    return {
+      verified: false,
+      reason: `Expected "${expectedEffect}" but the prior UI state still appears present`,
+    };
+  }
+
+  if (/(visible|appear|show|open|reveal|expand)/i.test(effectLower)) {
+    if (matchesKeywordDelta(expectedEffect, preActionState, postActionState)) {
+      return { verified: true };
+    }
+    if (addedLines.length > 0) {
+      return { verified: true };
+    }
+    return {
+      verified: false,
+      reason: `Expected "${expectedEffect}" but no new visible UI evidence appeared`,
+    };
+  }
+
+  if (/(switch|load|navigate|redirect)/i.test(effectLower)) {
+    if (postActionState.url !== preActionState.url || postActionState.title !== preActionState.title) {
+      return { verified: true };
+    }
+    if (matchesKeywordDelta(expectedEffect, preActionState, postActionState)) {
+      return { verified: true };
+    }
+    return {
+      verified: false,
+      reason: `Expected "${expectedEffect}" but the page identity did not materially change`,
+    };
+  }
+
+  if (matchesKeywordDelta(expectedEffect, preActionState, postActionState)) {
+    return { verified: true };
+  }
+
+  if (changed) {
+    return { verified: true };
+  }
+
+  return {
+    verified: false,
+    reason: `Expected effect "${expectedEffect}" — page did not change`,
+  };
+}

src/runner/index.ts

@@ -27,6 +27,9 @@ export {
   detectCompletionContentTypeMismatch,
 } from './goal-verification.js';
 
+// Effect verification
+export { verifyExpectedEffect } from './effect-verification.js';
+
 // Page analysis
 export {
   detectAiTangleVerifiedOutputState,