evals: allow env field in eval config .json which is passed to sdk run()

Author: jahooma

evals/buffbench/agent-runner.ts

@@ -19,6 +19,7 @@ export async function runAgentOnCommit({
   commit,
   repoUrl,
   initCommand,
+  env,
   localAgentDefinitions,
   printEvents,
 }: {
@@ -27,6 +28,7 @@ export async function runAgentOnCommit({
   commit: EvalCommitV2
   repoUrl: string
   initCommand?: string
+  env?: Record<string, string>
   localAgentDefinitions: any[]
   printEvents: boolean
 }): Promise<{
@@ -51,6 +53,7 @@ export async function runAgentOnCommit({
         repoUrl,
         parentSha: commit.parentSha,
         initCommand,
+        env,
       },
       async (repoDir) => {
         const timeoutMs = 30 * 60 * 1000 // 30 minutes
@@ -60,6 +63,7 @@ export async function runAgentOnCommit({
             prompt: commit.prompt,
             agentDefinitions: localAgentDefinitions,
             cwd: repoDir,
+            env,
             handleEvent: (event) => {
               if (
                 (event.type === 'tool_call' || event.type === 'tool_result') &&

evals/buffbench/eval-codebuff.json

@@ -2,6 +2,33 @@
   "repoUrl": "https://github.com/CodebuffAI/codebuff",
   "generationDate": "2025-10-12T05:55:40.855Z",
   "initCommand": "bun install",
+  "env": {
+    "ANTHROPIC_API_KEY": "test-key",
+    "ANTHROPIC_API_KEY2": "test-key-2",
+    "HELICONE_API_KEY": "test-helicone",
+    "OPEN_AI_KEY": "test-openai",
+    "GOOGLE_GENERATIVE_AI_API_KEY": "test-google",
+    "DEEPSEEK_API_KEY": "test-deepseek",
+    "GEMINI_API_KEY": "test-gemini",
+    "OPEN_ROUTER_API_KEY": "test-openrouter",
+    "RELACE_API_KEY": "test-relace",
+    "LINKUP_API_KEY": "test-linkup",
+    "GOOGLE_CLOUD_PROJECT_ID": "test-project",
+    "PORT": 3003,
+    "DATABASE_URL": "postgresql://test",
+    "CODEBUFF_GITHUB_ID": "test-github-id",
+    "CODEBUFF_GITHUB_SECRET": "test-github-secret",
+    "NEXTAUTH_SECRET": "test-nextauth-secret",
+    "STRIPE_SECRET_KEY": "test-stripe-key",
+    "STRIPE_WEBHOOK_SECRET_KEY": "test-stripe-webhook",
+    "STRIPE_USAGE_PRICE_ID": "test-price-id",
+    "STRIPE_TEAM_FEE_PRICE_ID": "test-team-price-id",
+    "LOOPS_API_KEY": "test-loops",
+    "DISCORD_PUBLIC_KEY": "test-discord-public",
+    "DISCORD_BOT_TOKEN": "test-discord-bot",
+    "DISCORD_APPLICATION_ID": "test-discord-app",
+    "API_KEY_ENCRYPTION_SECRET": "12345678901234567890123456789012"
+  },
   "evalCommits": [
     {
       "id": "filter-system-history",

evals/buffbench/run-buffbench.ts

@@ -22,6 +22,7 @@ async function runTask(options: {
   agents: string[]
   repoUrl: string
   initCommand?: string
+  env?: Record<string, string>
   logsDir: string
   index: number
   totalTasks: number
@@ -40,6 +41,7 @@ async function runTask(options: {
     agents,
     repoUrl,
     initCommand,
+    env,
     logsDir,
     index,
     totalTasks,
@@ -63,6 +65,7 @@ async function runTask(options: {
       commit,
       repoUrl,
       initCommand,
+      env,
       localAgentDefinitions,
       printEvents,
     })
@@ -300,6 +303,7 @@ export async function runBuffBench(options: {
         agents,
         repoUrl: evalData.repoUrl,
         initCommand: evalData.initCommand,
+        env: evalData.env,
         logsDir,
         index,
         totalTasks: commitsToRun.length,

evals/buffbench/types.ts

@@ -43,6 +43,7 @@ export interface EvalDataV2 {
   testRepoName?: string
   generationDate: string
   initCommand?: string
+  env?: Record<string, string>
   evalCommits: EvalCommitV2[]
 }
 

evals/subagents/test-repo-utils.ts

@@ -14,10 +14,11 @@ export const withTestRepo = async <T>(
     // The sha of the commit to checkout. If you have a commit with changes to replicate, you would check out the parent commit.
     parentSha: string
     initCommand?: string
+    env?: Record<string, string>
   },
   fn: (cwd: string) => Promise<T>,
 ): Promise<T> => {
-  const { repoUrl, parentSha, initCommand } = repoConfig
+  const { repoUrl, parentSha, initCommand, env } = repoConfig
 
   // Create a temporary directory for the test repo
   const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codebuff-eval-'))
@@ -35,7 +36,11 @@ export const withTestRepo = async <T>(
     if (initCommand) {
       console.log(`Running init command: ${initCommand}...`)
       try {
-        execSync(initCommand, { cwd: repoDir })
+        execSync(initCommand, {
+          cwd: repoDir,
+          stdio: 'ignore',
+          env: { ...process.env, ...env },
+        })
       } catch (error) {
         console.error(
           `Error running init command: ${getErrorObject(error).message}`,

sdk/src/tools/run-terminal-command.ts

@@ -40,9 +40,6 @@ export function runTerminalCommand({
       env: {
         ...process.env,
         ...(env ?? {}),
-        FORCE_COLOR: '1',
-        CLICOLOR: '1',
-        CLICOLOR_FORCE: '1',
       },
       stdio: 'pipe',
     })