evals: Add typecheck / tests for judge

Author: jahooma

evals/buffbench/agent-runner.ts

@@ -1,13 +1,17 @@
 import fs from 'fs'
 import path from 'path'
 import { execSync } from 'child_process'
+import { promisify } from 'util'
+import { exec } from 'child_process'
+
+const execAsync = promisify(exec)
 
 import { withTimeout } from '@codebuff/common/util/promise'
 import { CodebuffClient } from '../../sdk/src/client'
 import { withTestRepo } from '../subagents/test-repo-utils'
 
 import type { PrintModeEvent } from '@codebuff/common/types/print-mode'
-import type { EvalCommitV2 } from './types'
+import type { EvalCommitV2, FinalCheckOutput } from './types'
 
 export type AgentStep = PrintModeEvent
 
@@ -22,6 +26,7 @@ export async function runAgentOnCommit({
   env,
   localAgentDefinitions,
   printEvents,
+  finalCheckCommands,
 }: {
   client: CodebuffClient
   agentId: string
@@ -31,13 +36,15 @@ export async function runAgentOnCommit({
   env?: Record<string, string>
   localAgentDefinitions: any[]
   printEvents: boolean
+  finalCheckCommands?: string[]
 }): Promise<{
   diff: string
   contextFiles: Record<string, string>
   durationMs: number
   cost: number
   error?: string
   trace: AgentStep[]
+  finalCheckOutputs?: FinalCheckOutput[]
 }> {
   console.log(`[${commit.id}] Running agent ${agentId}...`)
   const startTime = Date.now()
@@ -46,6 +53,7 @@ export async function runAgentOnCommit({
   let error: string | undefined
   let cost = 0
   const trace: AgentStep[] = []
+  let finalCheckOutputs: FinalCheckOutput[] | undefined
 
   try {
     await withTestRepo(
@@ -140,6 +148,18 @@ export async function runAgentOnCommit({
             contextFiles[filePath] = ''
           }
         }
+
+        // Run final check commands if specified
+        if (finalCheckCommands && finalCheckCommands.length > 0) {
+          console.log(
+            `[${commit.id}] Running ${finalCheckCommands.length} final check commands...`,
+          )
+          finalCheckOutputs = await runFinalCheckCommands(
+            finalCheckCommands,
+            repoDir,
+            env,
+          )
+        }
       },
     )
   } catch (e) {
@@ -155,5 +175,44 @@ export async function runAgentOnCommit({
     cost,
     error,
     trace,
+    finalCheckOutputs,
   }
 }
+
+async function runFinalCheckCommands(
+  commands: string[],
+  cwd: string,
+  env?: Record<string, string>,
+): Promise<FinalCheckOutput[]> {
+  const results: FinalCheckOutput[] = []
+
+  for (const command of commands) {
+    console.log(`  Running: ${command}`)
+    try {
+      const { stdout, stderr } = await execAsync(command, {
+        cwd,
+        encoding: 'utf-8',
+        maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+        env: { ...process.env, ...env },
+      })
+      results.push({
+        command,
+        exitCode: 0,
+        stdout,
+        stderr,
+      })
+      console.log(`  ✓ Command succeeded: ${command}`)
+    } catch (error: any) {
+      // Command failed, but we still capture the output
+      results.push({
+        command,
+        exitCode: error.code || 1,
+        stdout: error.stdout || '',
+        stderr: error.stderr || error.message || '',
+      })
+      console.log(`  ✗ Command failed (exit ${error.code}): ${command}`)
+    }
+  }
+
+  return results
+}

evals/buffbench/eval-codebuff.json

@@ -9,6 +9,7 @@
     }
   ],
   "initCommand": "bun install",
+  "finalCheckCommands": ["bun run typecheck", "bun run test"],
   "env": {
     "ANTHROPIC_API_KEY": "test-key",
     "ANTHROPIC_API_KEY2": "test-key-2",

evals/buffbench/judge.ts

@@ -125,6 +125,7 @@ interface JudgeCommitResultInput {
   contextFiles: Record<string, string>
   agentDiff: string
   error?: string
+  finalCheckOutputs?: string
 }
 
 export async function judgeCommitResult(
@@ -137,6 +138,7 @@ export async function judgeCommitResult(
     contextFiles,
     agentDiff,
     error,
+    finalCheckOutputs,
   } = input
 
   const groundTruthDiffs = groundTruthFileDiffs
@@ -164,7 +166,8 @@ ${groundTruthDiffs}
 \`\`\`diff
 ${agentDiff || '(No changes made)'}
 \`\`\`
-${error ? `\n## Error Encountered\n${error}` : ''}`
+${error ? `\n## Error Encountered\n${error}` : ''}
+${finalCheckOutputs ? `\n## Final Check Command Outputs\n${finalCheckOutputs}` : ''}`
 
   const agentOutput: string[] = []
   const judgeResult = await withTimeout(

evals/buffbench/run-buffbench.ts

@@ -36,6 +36,7 @@ async function runTask(options: {
   localAgentDefinitions: any[]
   extractLessons: boolean
   printEvents: boolean
+  finalCheckCommands?: string[]
 }) {
   const {
     client,
@@ -51,6 +52,7 @@ async function runTask(options: {
     localAgentDefinitions,
     extractLessons,
     printEvents,
+    finalCheckCommands,
   } = options
 
   console.log(
@@ -70,6 +72,7 @@ async function runTask(options: {
       env,
       localAgentDefinitions,
       printEvents,
+      finalCheckCommands,
     })
 
     const judgeResult = await judgeCommitResult({
@@ -79,6 +82,14 @@ async function runTask(options: {
       contextFiles: agentResult.contextFiles,
       agentDiff: agentResult.diff,
       error: agentResult.error,
+      finalCheckOutputs: agentResult.finalCheckOutputs
+        ? agentResult.finalCheckOutputs
+            .map(
+              (output) =>
+                `### ${output.command}\n\`\`\`\n${output.stdout}${output.stderr ? '\nSTDERR:\n' + output.stderr : ''}\n\`\`\``,
+            )
+            .join('\n\n')
+        : undefined,
     })
 
     // Extract and append agent lessons
@@ -114,6 +125,7 @@ async function runTask(options: {
       cost: agentResult.cost,
       durationMs: agentResult.durationMs,
       error: agentResult.error,
+      finalCheckOutputs: agentResult.finalCheckOutputs,
     }
 
     // Save trace to logs directory
@@ -135,6 +147,7 @@ async function runTask(options: {
       durationMs: agentResult.durationMs,
       error: agentResult.error,
       timestamp: new Date().toISOString(),
+      finalCheckOutputs: agentResult.finalCheckOutputs,
     })
 
     fs.writeFileSync(
@@ -369,6 +382,7 @@ export async function runBuffBench(options: {
         localAgentDefinitions: analyzerContext.agentDefinitions,
         extractLessons,
         printEvents: agents.length === 1 && taskConcurrency === 1,
+        finalCheckCommands: evalData.finalCheckCommands,
       }),
     ),
   )

evals/buffbench/trace-analyzer.ts

@@ -6,6 +6,8 @@ import { withTimeout } from '@codebuff/common/util/promise'
 import { getErrorObject } from '@codebuff/common/util/error'
 import { truncateTrace } from './trace-utils'
 
+import type { FinalCheckOutput } from './types'
+
 export interface AgentTraceData {
   agentId: string
   commitSha: string
@@ -17,6 +19,7 @@ export interface AgentTraceData {
   durationMs: number
   error?: string
   timestamp: string
+  finalCheckOutputs?: FinalCheckOutput[]
 }
 
 const traceAnalyzerAgent: AgentDefinition = {

evals/buffbench/types.ts

@@ -51,9 +51,17 @@ export interface EvalDataV2 {
   initCommand?: string
   binInstalls?: BinInstall[]
   env?: Record<string, string>
+  finalCheckCommands?: string[]
   evalCommits: EvalCommitV2[]
 }
 
+export interface FinalCheckOutput {
+  command: string
+  exitCode: number
+  stdout: string
+  stderr: string
+}
+
 export interface EvalRun {
   commitSha: string
   prompt: string
@@ -62,6 +70,7 @@ export interface EvalRun {
   cost: number
   durationMs: number
   error?: string
+  finalCheckOutputs?: FinalCheckOutput[]
 }
 
 export interface AgentEvalResults {