feat: expand deep review fixture coverage

Broaden the frontier eval suite with new authz, async, and CI-injection regressions so live runs cover more high-signal failure modes. Harden semantic matching around tenant-isolation, async-await, and GitHub Actions phrasing so benchmarks reward the substance of correct findings across frontier models.

Made-with: Cursor

Author: haasonsaas

eval/fixtures/deep_review_suite/review_depth_async.json

@@ -115,6 +115,46 @@
       "max_total": 8,
       "description": "Awaiting async permission checks is required before destructive actions.",
       "source": "deep-review-suite"
+    },
+    {
+      "name": "typescript-async-foreach-not-awaited",
+      "category": "bug",
+      "language": "typescript",
+      "difficulty": "Hard",
+      "diff_content": "diff --git a/src/notify.ts b/src/notify.ts\nindex 1111111..2222222 100644\n--- a/src/notify.ts\n+++ b/src/notify.ts\n@@ -1,5 +1,7 @@\n export async function sendNotifications(users) {\n-  await Promise.all(users.map((user) => sendNotification(user)));\n+  users.forEach(async (user) => {\n+    await sendNotification(user);\n+  });\n   return { sent: true };\n }\n",
+      "expected_findings": [
+        {
+          "description": "Async work launched inside forEach is not awaited before the function returns.",
+          "severity": "Warning",
+          "category": "Bug",
+          "file_pattern": "src/notify.ts",
+          "line_hint": 2,
+          "contains_any": [
+            "foreach does not await",
+            "async callbacks are not awaited",
+            "returns before sendnotification completes",
+            "fire and forget",
+            "promise is ignored",
+            "map result is ignored",
+            "does not await promises",
+            "return before all notifications complete",
+            "breaks the async contract"
+          ],
+          "tags_any": [
+            "async"
+          ]
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Notification fan-out should keep awaiting all spawned async work.",
+      "source": "deep-review-suite"
     }
   ]
 }

eval/fixtures/deep_review_suite/review_depth_authz.json

@@ -93,6 +93,46 @@
       "max_total": 8,
       "description": "Privilege-gated destructive operations must keep their role checks.",
       "source": "deep-review-suite"
+    },
+    {
+      "name": "python-tenant-scope-removed-from-invoice-lookup",
+      "category": "security",
+      "language": "python",
+      "difficulty": "Hard",
+      "diff_content": "diff --git a/billing.py b/billing.py\nindex 1111111..2222222 100644\n--- a/billing.py\n+++ b/billing.py\n@@ -1,5 +1,5 @@\n def mark_invoice_paid(request, db):\n-    invoice = db.invoices.find_one({\"id\": request.args[\"invoice_id\"], \"tenant_id\": request.user.tenant_id})\n+    invoice = db.invoices.find_one({\"id\": request.args[\"invoice_id\"]})\n     invoice[\"status\"] = \"paid\"\n     db.invoices.save(invoice)\n     return {\"ok\": True}\n",
+      "expected_findings": [
+        {
+          "description": "Invoice lookup is no longer constrained to the caller's tenant.",
+          "severity": "Error",
+          "category": "Security",
+          "file_pattern": "billing.py",
+          "line_hint": 2,
+          "contains_any": [
+            "tenant isolation",
+            "cross-tenant access",
+            "missing tenant filter",
+            "authorization bypass",
+            "idor",
+            "invoice lookup is not scoped",
+            "removed tenant_id check",
+            "other tenants",
+            "broken authorization"
+          ],
+          "tags_any": [
+            "authorization"
+          ]
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Multi-tenant invoice access must stay scoped to the caller's tenant.",
+      "source": "deep-review-suite"
     }
   ]
 }

eval/fixtures/deep_review_suite/review_depth_supply_chain.json

@@ -108,6 +108,61 @@
       "max_total": 8,
       "description": "Build pipelines should not execute unverified remote scripts.",
       "source": "deep-review-suite"
+    },
+    {
+      "name": "yaml-pull-request-target-script-injection",
+      "category": "security",
+      "language": "yaml",
+      "difficulty": "Expert",
+      "diff_content": "diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml\nindex 1111111..2222222 100644\n--- a/.github/workflows/pr-check.yml\n+++ b/.github/workflows/pr-check.yml\n@@ -1,5 +1,8 @@\n name: PR Check\n-on: pull_request\n+on: pull_request_target\n jobs:\n   test:\n     runs-on: ubuntu-latest\n+    steps:\n+      - run: echo \"${{ github.event.pull_request.title }}\" | bash\n",
+      "expected_findings": [
+        {
+          "description": "pull_request_target can expose write-scoped credentials to untrusted PR code.",
+          "severity": "Warning",
+          "category": "Security",
+          "file_pattern": ".github/workflows/pr-check.yml",
+          "line_hint": 2,
+          "contains_any": [
+            "pull_request_target",
+            "untrusted code with write permissions",
+            "elevated github token",
+            "ci injection"
+          ],
+          "tags_any": [
+            "supply-chain"
+          ],
+          "rule_id": "sec.supply-chain.ci-injection"
+        },
+        {
+          "description": "Attacker-controlled GitHub event data is piped into bash.",
+          "severity": "Error",
+          "category": "Security",
+          "file_pattern": ".github/workflows/pr-check.yml",
+          "line_hint": 7,
+          "contains_any": [
+            "script injection via github.event",
+            "attacker-controlled pr title",
+            "piped into bash",
+            "untrusted github event context",
+            "pr title is piped directly to bash",
+            "arbitrary command execution"
+          ],
+          "tags_any": [
+            "supply-chain"
+          ],
+          "rule_id": "sec.supply-chain.ci-injection"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "CI workflows should not combine pull_request_target with attacker-controlled shell input.",
+      "source": "deep-review-suite"
     }
   ]
 }

src/commands/eval/pattern/matching/predicates.rs

@@ -239,6 +239,11 @@ fn category_aliases(expected: &str) -> &'static [&'static str] {
             "secret",
             "forbidden",
             "unauthorized",
+            "tenant isolation",
+            "cross tenant",
+            "cross tenant access",
+            "multi tenant",
+            "other tenants",
         ],
         "bug" => &[
             "bug",
@@ -251,8 +256,11 @@ fn category_aliases(expected: &str) -> &'static [&'static str] {
             "background task",
             "spawned task",
             "not awaited",
+            "does not await",
             "missing await",
             "promise is always truthy",
+            "async contract",
+            "unhandled promise",
             "swallowed error",
             "logic error",
             "race condition",
@@ -289,6 +297,10 @@ fn canonicalize_semantic_text(text: &str) -> String {
         ("authorisation", "authorization"),
         ("access control", "authorization"),
         ("broken access control", "authorization bypass"),
+        ("cross-tenant", "cross tenant"),
+        ("multi-tenancy", "tenant isolation"),
+        ("multi tenancy", "tenant isolation"),
+        ("tenant_id", "tenant"),
         ("verbose-error", "information disclosure"),
         ("verbose error", "information disclosure"),
         ("debug-details", "information disclosure"),
@@ -313,6 +325,8 @@ fn canonicalize_semantic_text(text: &str) -> String {
         ("attack vector", "risk"),
         ("silently discarded", "swallowed error"),
         ("silent failure", "swallowed error"),
+        ("piped directly to bash", "piped into bash"),
+        ("piped to bash", "piped into bash"),
         ("sqli", "sql injection"),
         ("xss", "cross site scripting"),
         ("ssrf", "server side request forgery"),
@@ -359,6 +373,9 @@ fn semantic_tokens(text: &str) -> Vec<String> {
             "auth" | "authz" => "authorization".to_string(),
             "authn" => "authentication".to_string(),
             "access" => "authorization".to_string(),
+            "tenants" => "tenant".to_string(),
+            "promises" => "promise".to_string(),
+            "callbacks" => "callback".to_string(),
             "piping" | "piped" | "pipes" => "pipe".to_string(),
             "bash" | "sh" => "shell".to_string(),
             "downloaded" | "downloading" | "downloads" => "download".to_string(),

src/commands/eval/pattern/matching/run.rs

@@ -243,6 +243,72 @@ mod tests {
         assert!(pattern.matches(&comment));
     }
 
+    #[test]
+    fn test_eval_pattern_matches_bug_category_from_async_foreach_signals() {
+        let comment = core::Comment {
+            id: "comment-3f".to_string(),
+            file_path: PathBuf::from("src/notify.ts"),
+            line_number: 2,
+            content: "forEach with async callback does not await promises, so the function returns before all notifications complete and breaks the async contract."
+                .to_string(),
+            rule_id: Some("async.foreach-no-await".to_string()),
+            severity: Severity::Error,
+            category: Category::BestPractice,
+            suggestion: None,
+            confidence: 0.9,
+            code_suggestion: None,
+            tags: vec!["async-await".to_string(), "promise".to_string()],
+            fix_effort: FixEffort::Low,
+            feedback: None,
+        };
+
+        let pattern = EvalPattern {
+            category: Some("bug".to_string()),
+            contains_any: vec![
+                "does not await promises".to_string(),
+                "returns before all notifications complete".to_string(),
+            ],
+            severity: Some("warning".to_string()),
+            tags_any: vec!["async".to_string()],
+            ..Default::default()
+        };
+
+        assert!(pattern.matches(&comment));
+    }
+
+    #[test]
+    fn test_eval_pattern_matches_security_category_from_tenant_isolation_signals() {
+        let comment = core::Comment {
+            id: "comment-3g".to_string(),
+            file_path: PathBuf::from("billing.py"),
+            line_number: 2,
+            content: "Removed tenant_id check from invoice query, allowing users to access invoices from other tenants."
+                .to_string(),
+            rule_id: Some("sec.authz.missing-tenant-check".to_string()),
+            severity: Severity::Error,
+            category: Category::Bug,
+            suggestion: None,
+            confidence: 0.95,
+            code_suggestion: None,
+            tags: vec!["multi-tenancy".to_string(), "authorization".to_string()],
+            fix_effort: FixEffort::Low,
+            feedback: None,
+        };
+
+        let pattern = EvalPattern {
+            category: Some("security".to_string()),
+            contains_any: vec![
+                "other tenants".to_string(),
+                "removed tenant_id check".to_string(),
+            ],
+            severity: Some("error".to_string()),
+            tags_any: vec!["authorization".to_string()],
+            ..Default::default()
+        };
+
+        assert!(pattern.matches(&comment));
+    }
+
     #[test]
     fn test_eval_pattern_matches_adjacent_line_hint() {
         let comment = core::Comment {