Use shallow clone

Author: jahooma

evals/git-evals2/agent-runner.ts

@@ -36,9 +36,8 @@ export async function runAgentOnCommit({
     await withTestRepo(
       {
         repoUrl,
-        commitSha: commit.sha,
+        parentSha: commit.parentSha,
         initCommand,
-        checkoutPrevious: true,
       },
       async (repoDir) => {
         const agentsPath = path.join(__dirname, '../../.agents')

evals/subagents/eval-planner.ts

@@ -15,7 +15,7 @@ export const evalPlannerAgent = async (params: {
   agentDefinitions: Array<AgentDefinition>
   spec: string
   repoUrl: string
-  commitSha: string
+  parentSha: string
   initCommand?: string
   fileStates: Array<{
     path: string
@@ -29,13 +29,13 @@ export const evalPlannerAgent = async (params: {
     agentDefinitions,
     spec,
     repoUrl,
-    commitSha,
+    parentSha,
     initCommand,
     fileStates,
   } = params
   const plannerStartTime = Date.now()
   const result = await withTestRepo(
-    { repoUrl, commitSha, initCommand, checkoutPrevious: true },
+    { repoUrl, parentSha, initCommand },
     async (cwd) => {
       // Run the agent with the test repository as cwd
       console.log(`Running agent ${agentId} with prompt: ${spec}...`)
@@ -206,6 +206,7 @@ type EvalData = {
   initCommand?: string
   evalCommits: Array<{
     sha: string
+    parentSha: string
     spec: string
     fileStates: Array<{
       path: string
@@ -262,7 +263,7 @@ async function main() {
 
   // Loop through each eval task
   for (const evalCommit of evalCommits) {
-    const { sha, spec, fileStates } = evalCommit
+    const { sha, parentSha, spec, fileStates } = evalCommit
 
     console.log(`\n=== Running eval for commit ${sha} ===`)
     console.log(`Spec: ${spec.substring(0, 100)}...\n`)
@@ -274,7 +275,7 @@ async function main() {
         agentDefinitions: localAgentDefinitions,
         spec,
         repoUrl,
-        commitSha: sha,
+        parentSha,
         initCommand,
         fileStates,
       })
@@ -372,7 +373,8 @@ async function main() {
 
   if (stats.plannerLatencies.length > 0) {
     const avgPlannerLatency =
-      stats.plannerLatencies.reduce((a, b) => a + b, 0) / stats.plannerLatencies.length
+      stats.plannerLatencies.reduce((a, b) => a + b, 0) /
+      stats.plannerLatencies.length
     const minPlannerLatency = Math.min(...stats.plannerLatencies)
     const maxPlannerLatency = Math.max(...stats.plannerLatencies)
     const medianPlannerLatency = stats.plannerLatencies.sort((a, b) => a - b)[

evals/subagents/test-repo-utils.ts

@@ -10,37 +10,30 @@ import { execSync } from 'child_process'
 export const withTestRepo = async <T>(
   repoConfig: {
     repoUrl: string
-    commitSha: string
+    // The sha of the commit to checkout. If you have a commit with changes to replicate, you would check out the parent commit.
+    parentSha: string
     initCommand?: string
-    checkoutPrevious?: boolean
   },
   fn: (cwd: string) => Promise<T>,
 ): Promise<T> => {
-  const { repoUrl, commitSha, initCommand, checkoutPrevious } = repoConfig
+  const { repoUrl, parentSha, initCommand } = repoConfig
 
   // Create a temporary directory for the test repo
   const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codebuff-eval-'))
   const repoDir = path.join(tempDir, 'repo')
 
   try {
-    // Clone the repository
-    console.log(`Cloning repository ${repoUrl} to ${repoDir}...`)
-    execSync(`git clone ${repoUrl} ${repoDir}`, { stdio: 'ignore' })
+    console.log(
+      `Cloning repository ${repoUrl} at commit ${parentSha} to ${repoDir} (shallow)...`,
+    )
+    execSync(`git clone --depth 1 ${repoUrl} ${repoDir}`, { stdio: 'ignore' })
 
-    // Checkout the specific commit or the previous commit
-    if (checkoutPrevious) {
-      const previousCommitSha = getPreviousCommitSha(commitSha, repoDir)
-      console.log(`Checking out previous commit ${previousCommitSha}...`)
-      execSync(`git checkout ${previousCommitSha}`, {
-        cwd: repoDir,
-        stdio: 'ignore',
-      })
-    } else {
-      console.log(`Checking out commit ${commitSha}...`)
-      execSync(`git checkout ${commitSha}`, { cwd: repoDir, stdio: 'ignore' })
-    }
+    execSync(`git fetch --depth 1 origin ${parentSha}`, {
+      cwd: repoDir,
+      stdio: 'ignore',
+    })
+    execSync(`git checkout ${parentSha}`, { cwd: repoDir, stdio: 'ignore' })
 
-    // Run initialization command if provided
     if (initCommand) {
       console.log(`Running init command: ${initCommand}...`)
       execSync(initCommand, { cwd: repoDir, stdio: 'ignore' })
@@ -50,22 +43,10 @@ export const withTestRepo = async <T>(
     return await fn(repoDir)
   } finally {
     // Clean up the temporary directory
-    console.log(`Cleaning up temporary directory ${tempDir}...`)
     try {
       fs.rmSync(tempDir, { recursive: true, force: true })
     } catch (error) {
       console.warn(`Failed to clean up temporary directory: ${error}`)
     }
   }
 }
-
-/**
- * Gets the previous commit SHA (parent) of a given commit
- */
-const getPreviousCommitSha = (commitSha: string, repoDir: string): string => {
-  const previousSha = execSync(`git rev-parse ${commitSha}^`, {
-    cwd: repoDir,
-    encoding: 'utf-8',
-  }).trim()
-  return previousSha
-}