test(eval): add fix-loop regression fixtures

Author: haasonsaas

TODO.md

@@ -57,7 +57,7 @@ This roadmap is derived from deep research into Greptile's public docs, blog, MC
 27. [x] Add "challenge the finding" verification loops where a validator tries to falsify a suspected issue before keeping it.
 28. [ ] Add caching between iterations so repeated codebase retrieval and verification runs are cheaper.
 29. [x] Allow loop policies to differ by profile: conservative auditor, high-autonomy fixer, or report-only.
-30. [ ] Add eval fixtures specifically for loop convergence and reopened-issue regressions.
+30. [x] Add eval fixtures specifically for loop convergence and reopened-issue regressions.
 
 ## 4. Code Graph and Repository Intelligence
 

eval/fixtures/repo_regressions/fix_loop_premature_convergence.yml

@@ -0,0 +1,28 @@
+name: repo regression - fix loop premature convergence
+repo_path: ../../..
+diff: |
+  diff --git a/src/server/api/gh.rs b/src/server/api/gh.rs
+  index eaf02e9..deadbeef 100644
+  --- a/src/server/api/gh.rs
+  +++ b/src/server/api/gh.rs
+  @@ -1949,8 +1949,7 @@ pub(crate) async fn run_gh_pr_fix_loop(
+       })?;
+  
+       if latest_summary_ref.merge_readiness == MergeReadiness::Ready
+           && latest_summary_ref.open_blockers == 0
+  -        && latest_summary_ref.open_comments == 0
+       {
+           return Ok(Json(build_pr_fix_loop_response(PrFixLoopResponseArgs {
+               repo: request.repo.clone(),
+expect:
+  must_find:
+    - file: src/server/api/gh.rs
+      contains_any:
+        - prematurely marks the fix loop as converged
+        - unresolved findings can still remain when the loop stops
+        - missing open_comments check in the convergence guard
+      rule_id: bug.fix-loop.premature-convergence
+  must_not_find:
+    - contains: style
+  min_total: 1
+  max_total: 8

eval/fixtures/repo_regressions/fix_loop_reopened_findings.yml

@@ -0,0 +1,28 @@
+name: repo regression - reopened finding telemetry broken
+repo_path: ../../..
+diff: |
+  diff --git a/src/server/api/gh.rs b/src/server/api/gh.rs
+  index eaf02e9..cafef00d 100644
+  --- a/src/server/api/gh.rs
+  +++ b/src/server/api/gh.rs
+  @@ -1075,8 +1075,8 @@ pub(crate) fn build_fix_loop_telemetry(
+           findings_cleared += previous_findings.difference(&current_findings).count();
+           findings_reopened += current_findings
+               .difference(&previous_findings)
+  -            .filter(|finding_id| historical_findings.contains(*finding_id))
+  +            .filter(|finding_id| previous_findings.contains(*finding_id))
+               .count();
+  
+           historical_findings.extend(current_findings);
+expect:
+  must_find:
+    - file: src/server/api/gh.rs
+      contains_any:
+        - reopened findings will never be counted correctly
+        - logic bug in reopened finding telemetry
+        - a finding in current minus previous can never also be in previous_findings
+      rule_id: bug.fix-loop.reopened-finding-telemetry
+  must_not_find:
+    - contains: style
+  min_total: 1
+  max_total: 8

src/commands/eval/fixtures.rs

@@ -172,6 +172,58 @@ expect:
             .any(|phrase| phrase.contains("trait implementation")));
     }
 
+    #[test]
+    fn test_checked_in_fix_loop_convergence_fixture_loads_expected_fields() {
+        let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("eval/fixtures/repo_regressions/fix_loop_premature_convergence.yml");
+
+        let fixtures = load_eval_fixtures_from_path(&fixture_path).unwrap();
+
+        assert_eq!(fixtures.len(), 1);
+        assert_eq!(
+            fixtures[0].fixture.name.as_deref(),
+            Some("repo regression - fix loop premature convergence")
+        );
+        assert_eq!(
+            fixtures[0].fixture.repo_path,
+            Some(std::path::PathBuf::from("../../.."))
+        );
+        assert_eq!(
+            fixtures[0].fixture.expect.must_find[0].file.as_deref(),
+            Some("src/server/api/gh.rs")
+        );
+        assert_eq!(
+            fixtures[0].fixture.expect.must_find[0].rule_id.as_deref(),
+            Some("bug.fix-loop.premature-convergence")
+        );
+    }
+
+    #[test]
+    fn test_checked_in_fix_loop_reopened_fixture_loads_expected_fields() {
+        let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("eval/fixtures/repo_regressions/fix_loop_reopened_findings.yml");
+
+        let fixtures = load_eval_fixtures_from_path(&fixture_path).unwrap();
+
+        assert_eq!(fixtures.len(), 1);
+        assert_eq!(
+            fixtures[0].fixture.name.as_deref(),
+            Some("repo regression - reopened finding telemetry broken")
+        );
+        assert_eq!(
+            fixtures[0].fixture.repo_path,
+            Some(std::path::PathBuf::from("../../.."))
+        );
+        assert_eq!(
+            fixtures[0].fixture.expect.must_find[0].file.as_deref(),
+            Some("src/server/api/gh.rs")
+        );
+        assert_eq!(
+            fixtures[0].fixture.expect.must_find[0].rule_id.as_deref(),
+            Some("bug.fix-loop.reopened-finding-telemetry")
+        );
+    }
+
     #[test]
     fn test_collect_eval_fixtures_expands_pack_entries_in_sorted_order() {
         let dir = tempdir().unwrap();