evals: write trace to file when run in 'dev'

Author: jahooma

evals/git-evals/run-git-evals.ts

@@ -233,14 +233,26 @@ Explain your reasoning in detail. Do not ask Codebuff to git commit changes.`,
     const judgingResults = await judgeEvalRun(evalRun)
     console.log('Judging results:', judgingResults)
 
-    return {
+    const result = {
       ...evalRun,
       judging_results: judgingResults,
       computed_metrics: {
         runtime_sec: durationMs / 1000,
         cost_usd: totalCostUsd,
       },
     }
+
+    if (process.env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev') {
+      const { eval_commit, gitDiff, ...rest } = result
+      const { fileStates, ...rest2 } = eval_commit
+
+      writeJsonToFile(
+        { ...rest, ...rest2 },
+        path.join(__dirname, `trace-${evalCommit.sha}.json`),
+      )
+    }
+
+    return result
   } catch (judgingError) {
     console.error('Error in judging:', judgingError)
     // Return without judging results if judging fails
@@ -269,6 +281,10 @@ Explain your reasoning in detail. Do not ask Codebuff to git commit changes.`,
   }
 }
 
+function writeJsonToFile(json: any, path: string) {
+  fs.writeFileSync(path, JSON.stringify(json, null, 2))
+}
+
 function getCodebuffFileStates(
   evalCommitSha: string,
   projectPath: string,