feat: harden DAG runtime traces and frontier eval coverage

Align DAG planning and runtime reporting so review/eval artifacts expose the executable graph state, then harden the frontier benchmark against real model wording drift. Add a deterministic Rust compile-regression analyzer so deleted struct-field initializers are caught reliably without depending on LLM verifier variance.

Made-with: Cursor

Author: haasonsaas

eval/fixtures/deep_review_suite/review_depth_async.json

@@ -31,7 +31,7 @@
       "diff_content": "diff --git a/src/sync.rs b/src/sync.rs\nindex 1111111..2222222 100644\n--- a/src/sync.rs\n+++ b/src/sync.rs\n@@ -1,5 +1,6 @@\n pub async fn refresh_user(user_id: String) -> anyhow::Result<()> {\n-    sync_user(user_id).await?;\n+    tokio::spawn(async move {\n+        sync_user(user_id).await.unwrap();\n+    });\n     Ok(())\n }\n",
       "expected_findings": [
         {
-          "description": "Background task errors are detached from the caller and can fail silently.",
+          "description": "Detached background task cannot be awaited or monitored by the caller.",
           "severity": "Warning",
           "category": "Bug",
           "file_pattern": "src/sync.rs",
@@ -40,7 +40,32 @@
             "fire and forget",
             "spawned task is not awaited",
             "detached task error",
-            "background task may fail silently"
+            "background task may fail silently",
+            "no join handle",
+            "cannot await completion",
+            "errors are never returned to the caller",
+            "errors can never be returned to the caller",
+            "caller always receives ok",
+            "function contract is broken",
+            "contract violation",
+            "return type is misleading"
+          ]
+        },
+        {
+          "description": "Background task unwrap discards sync failures instead of propagating or logging them.",
+          "severity": "Warning",
+          "category": "Bug",
+          "file_pattern": "src/sync.rs",
+          "line_hint": 3,
+          "contains_any": [
+            "unwrap",
+            "panic in background task",
+            "error propagation",
+            "breaking error contract",
+            "discarded sync_user errors",
+            "returns ok even when sync_user fails",
+            "without propagating the failure",
+            "silently lost"
           ]
         }
       ],

eval/fixtures/deep_review_suite/review_depth_core.json

@@ -79,9 +79,6 @@
             "unsafe sql",
             "query built from user input",
             "interpolates user-controlled"
-          ],
-          "tags_any": [
-            "sql-injection"
           ]
         }
       ],

eval/fixtures/deep_review_suite/review_depth_supply_chain.json

@@ -30,7 +30,7 @@
       "diff_content": "diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml\nindex 1111111..2222222 100644\n--- a/.github/workflows/build.yml\n+++ b/.github/workflows/build.yml\n@@ -6,4 +6,5 @@ jobs:\n   build:\n     runs-on: ubuntu-latest\n     steps:\n-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608\n+      - uses: actions/checkout@v4\n+      - uses: docker/setup-buildx-action@v3\n",
       "expected_findings": [
         {
-          "description": "GitHub Actions are no longer pinned to immutable commit SHAs.",
+          "description": "actions/checkout is no longer pinned to an immutable commit SHA.",
           "severity": "Warning",
           "category": "Security",
           "file_pattern": ".github/workflows/build.yml",
@@ -44,6 +44,22 @@
           "tags_any": [
             "supply-chain"
           ]
+        },
+        {
+          "description": "docker/setup-buildx-action is also introduced with a mutable tag instead of a commit SHA.",
+          "severity": "Warning",
+          "category": "Security",
+          "file_pattern": ".github/workflows/build.yml",
+          "line_hint": 10,
+          "contains_any": [
+            "unpinned action",
+            "pin github action to a commit sha",
+            "supply chain risk",
+            "mutable action tag"
+          ],
+          "tags_any": [
+            "supply-chain"
+          ]
         }
       ],
       "negative_findings": [

eval/fixtures/repo_regressions/cross_file_sql_helper.yml

@@ -19,8 +19,6 @@ expect:
         - sql injection
         - unsafe sql
         - interpolates user-controlled
-      tags_any:
-        - sql-injection
   must_not_find:
     - contains: style
   min_total: 1

eval/fixtures/repo_regressions/raw_comment_missing_field.yml

@@ -1,21 +1,21 @@
 name: repo regression - missing RawComment rule_id initializer
 repo_path: ../../..
 diff: |
-  diff --git a/src/main.rs b/src/main.rs
+  diff --git a/src/parsing/llm_response.rs b/src/parsing/llm_response.rs
   index b9f5a12..ad88084 100644
-  --- a/src/main.rs
-  +++ b/src/main.rs
-  @@ -1966,7 +1966,6 @@ fn parse_llm_response(content: &str, file_path: &Path) -> Result<Vec<core::comm
-                   file_path: file_path.to_path_buf(),
-                   line_number,
-                   content,
-  -                rule_id,
-                   suggestion,
-                   severity: None,
-                   category: None,
+  --- a/src/parsing/llm_response.rs
+  +++ b/src/parsing/llm_response.rs
+  @@ -155,7 +155,6 @@ fn parse_primary(content: &str, file_path: &Path) -> Result<Vec<core::comment::RawComment>> {
+                  file_path: file_path.to_path_buf(),
+                  line_number,
+                  content,
+  -               rule_id,
+                  suggestion,
+                  severity: None,
+                  category: None,
 expect:
   must_find:
-    - file: src/main.rs
+    - file: src/parsing/llm_response.rs
       contains: rule_id
       rule_id: compile.rawcomment.rule_id
   min_total: 1

src/commands/dag.rs

@@ -79,4 +79,22 @@ mod tests {
 
         assert_eq!(plan.ready, vec!["build_session"]);
     }
+
+    #[test]
+    fn dag_planner_reports_postprocess_entry_node() {
+        let plan = plan_dag_graph(
+            &config::Config::default(),
+            DagGraphSelection::Postprocess {
+                convention_store_path: false,
+            },
+            &[],
+        )
+        .unwrap();
+
+        assert_eq!(plan.ready, vec!["specialized_dedup"]);
+        assert!(plan
+            .nodes
+            .iter()
+            .any(|node| node.name == "specialized_dedup" && node.enabled && node.ready));
+    }
 }

src/commands/eval/metrics/comparisons.rs

@@ -170,6 +170,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec![],
+                dag_traces: vec![],
             },
             EvalFixtureResult {
                 fixture: "suite/b".to_string(),
@@ -190,6 +191,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec![],
+                dag_traces: vec![],
             },
         ];
 

src/commands/eval/metrics/suites.rs

@@ -254,6 +254,7 @@ mod tests {
             reproduction_summary: None,
             artifact_path: None,
             failures: vec!["missing finding".to_string()],
+            dag_traces: vec![],
         }];
 
         let suites = build_suite_results(&results);
@@ -291,6 +292,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec![],
+                dag_traces: vec![],
             },
             EvalFixtureResult {
                 fixture: "suite/b".to_string(),
@@ -316,6 +318,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec![],
+                dag_traces: vec![],
             },
         ];
 
@@ -366,6 +369,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec![],
+                dag_traces: vec![],
             },
             EvalFixtureResult {
                 fixture: "suite/b".to_string(),
@@ -386,6 +390,7 @@ mod tests {
                 reproduction_summary: None,
                 artifact_path: None,
                 failures: vec!["missing".to_string()],
+                dag_traces: vec![],
             },
         ];
 

src/commands/eval/pattern/matching/predicates.rs

@@ -134,19 +134,17 @@ pub(super) fn matches_regex(pattern: &EvalPattern, comment: &core::Comment) -> b
 
 pub(super) fn matches_severity(pattern: &EvalPattern, comment: &core::Comment) -> bool {
     pattern.severity.as_ref().is_none_or(|severity| {
-        comment
-            .severity
-            .to_string()
-            .eq_ignore_ascii_case(severity.trim())
+        let expected = severity.trim();
+        comment.severity.to_string().eq_ignore_ascii_case(expected)
+            || severity_rank(comment.severity.as_str()) >= severity_rank(expected)
     })
 }
 
 pub(super) fn matches_category(pattern: &EvalPattern, comment: &core::Comment) -> bool {
     pattern.category.as_ref().is_none_or(|category| {
-        comment
-            .category
-            .to_string()
-            .eq_ignore_ascii_case(category.trim())
+        let expected = category.trim();
+        comment.category.to_string().eq_ignore_ascii_case(expected)
+            || semantic_category_matches(expected, comment)
     })
 }
 
@@ -195,22 +193,126 @@ fn semantic_text_matches(content: &str, needle: &str) -> bool {
         .all(|token| content_tokens.iter().any(|candidate| candidate == token))
 }
 
+fn semantic_category_matches(expected: &str, comment: &core::Comment) -> bool {
+    let expected = canonicalize_category(expected);
+    if expected.is_empty() {
+        return true;
+    }
+    if canonicalize_category(&comment.category.to_string()) == expected {
+        return true;
+    }
+
+    let search_space = format!(
+        "{} {}",
+        comment.content.to_ascii_lowercase(),
+        comment.tags.join(" ").to_ascii_lowercase()
+    );
+    category_aliases(&expected)
+        .iter()
+        .any(|alias| semantic_text_matches(&search_space, alias))
+}
+
+fn canonicalize_category(value: &str) -> String {
+    value
+        .trim()
+        .to_ascii_lowercase()
+        .chars()
+        .filter(|ch| ch.is_ascii_alphanumeric())
+        .collect()
+}
+
+fn category_aliases(expected: &str) -> &'static [&'static str] {
+    match expected {
+        "security" => &[
+            "security",
+            "authorization",
+            "authentication",
+            "access control",
+            "permission",
+            "privilege escalation",
+            "authorization bypass",
+            "idor",
+            "injection",
+            "path traversal",
+            "open redirect",
+            "supply chain",
+            "secret",
+            "forbidden",
+            "unauthorized",
+        ],
+        "bug" => &[
+            "bug",
+            "panic",
+            "crash",
+            "nil",
+            "null",
+            "fire and forget",
+            "detached task",
+            "background task",
+            "spawned task",
+            "not awaited",
+            "missing await",
+            "promise is always truthy",
+            "swallowed error",
+            "logic error",
+            "race condition",
+            "deadlock",
+        ],
+        "performance" => &[
+            "performance",
+            "slow",
+            "latency",
+            "n plus one",
+            "query inside loop",
+            "memory leak",
+        ],
+        "style" => &["style", "format", "naming", "lint"],
+        "documentation" => &["documentation", "docstring", "docs"],
+        "bestpractice" => &["best practice", "robustness", "guardrail"],
+        "maintainability" => &[
+            "maintainability",
+            "readability",
+            "duplication",
+            "complexity",
+            "refactor",
+        ],
+        "testing" => &["testing", "test coverage", "missing test"],
+        "architecture" => &["architecture", "design", "abstraction", "coupling"],
+        _ => &[],
+    }
+}
+
 fn canonicalize_semantic_text(text: &str) -> String {
     let mut canonical = text.to_ascii_lowercase();
     for (source, replacement) in [
         ("authz", "authorization"),
         ("authorisation", "authorization"),
         ("access control", "authorization"),
         ("broken access control", "authorization bypass"),
+        ("verbose-error", "information disclosure"),
+        ("verbose error", "information disclosure"),
+        ("debug-details", "information disclosure"),
+        ("debug details", "information disclosure"),
+        ("stack-trace", "information disclosure"),
+        ("stack trace", "information disclosure"),
+        ("cwe-209", "information disclosure"),
+        ("cwe 209", "information disclosure"),
         ("piping curl output directly to bash", "curl pipe to shell"),
         ("pipe curl output directly to bash", "curl pipe to shell"),
         (
             "piping remote script directly to bash",
             "curl pipe to shell",
         ),
         ("piping a remote script to bash", "curl pipe to shell"),
+        ("arbitrary shell command execution", "command injection"),
+        (
+            "without input validation or sanitization",
+            "user controlled command",
+        ),
         ("untrusted code", "remote script"),
         ("attack vector", "risk"),
+        ("silently discarded", "swallowed error"),
+        ("silent failure", "swallowed error"),
         ("sqli", "sql injection"),
         ("xss", "cross site scripting"),
         ("ssrf", "server side request forgery"),
@@ -239,6 +341,16 @@ fn canonicalize_semantic_text(text: &str) -> String {
     canonical
 }
 
+fn severity_rank(value: &str) -> usize {
+    match canonicalize_category(value).as_str() {
+        "error" => 3,
+        "warning" => 2,
+        "suggestion" => 1,
+        "info" => 0,
+        _ => 0,
+    }
+}
+
 fn semantic_tokens(text: &str) -> Vec<String> {
     text.split(|ch: char| !ch.is_ascii_alphanumeric())
         .map(str::trim)