evals: remove efficiency metric

Author: charleslien

evals/git-evals/email-eval-results.ts

@@ -33,11 +33,6 @@ function formatEvalSummaryForEmail(
       (sum, result) => sum + result.overall_metrics.average_completion,
       0,
     ) / evalResults.length
-  const avgEfficiency =
-    evalResults.reduce(
-      (sum, result) => sum + result.overall_metrics.average_efficiency,
-      0,
-    ) / evalResults.length
   const avgCodeQuality =
     evalResults.reduce(
       (sum, result) => sum + result.overall_metrics.average_code_quality,
@@ -70,14 +65,12 @@ function formatEvalSummaryForEmail(
 • Success Rate: ${successfulRuns}/${totalRuns} (${((successfulRuns / totalRuns) * 100).toFixed(1)}%)
 • Overall Score: ${avgOverallScore.toFixed(2)}/10
 • Completion: ${avgCompletion.toFixed(2)}/10
-• Efficiency: ${avgEfficiency.toFixed(2)}/10
 • Code Quality: ${avgCodeQuality.toFixed(2)}/10
 
 💰 COST & PERFORMANCE METRICS
 • Average Cost per Run: ${avgCostUsd.toFixed(4)}
 • Total Cost: ${totalCostUsd.toFixed(2)}
 • Average Runtime: ${avgRuntimeSec.toFixed(1)} seconds
-• Cost per Point (Overall Score): ${(avgCostUsd / avgOverallScore).toFixed(4)}
 
 📈 BY EVAL SET:
 ${evalResults

evals/git-evals/judge-git-eval.ts

@@ -55,14 +55,6 @@ function buildAnalysisPrompt(
 ${evalRun.eval_commit.spec}
 [/SPEC]
 
-[TIMING_INFORMATION]
-Task Duration: ${durationSeconds} seconds (${evalRun.durationMs}ms)
-[/TIMING_INFORMATION]
-
-[COST_INFORMATION]
-Total Cost: ${evalRun.costUsd.toFixed(2)} USD
-[/COST_INFORMATION]
-
 [GROUND_TRUTH_CHANGES]
 ${groundTruthChanges}
 [/GROUND_TRUTH_CHANGES]
@@ -71,10 +63,6 @@ ${groundTruthChanges}
 ${codebuffChanges}
 [/CHANGES_BY_CODEBUFF]
 
-[TRACE]
-${traceContent}
-[/TRACE]
-
 [ERROR]
 ${evalRun.error ? evalRun.error : 'None'}
 [/ERROR]
@@ -84,15 +72,13 @@ Please analyze the trace of the implementation attempt and provide:
 2. Key strengths and weaknesses of the implementation
 3. Numerical scores (0-10):
    - Completion: How completely and correctly was the spec implemented compared to the ground truth changes?
-   - Efficiency: How efficiently did Codebuff respond to the Agent's prompts without taking unnecessary steps? Speed is important! Consider the task duration of ${durationSeconds} seconds.
    - Code Quality: How well-structured, maintainable and idiomatic is the code?
    - Overall: Combined assessment of the implementation quality
 
 Focus on:
 - Correctness and completeness compared to the ground truth changes
 - Quality of the code produced
 - Minimal changes: it's better to change as little code as possible to accomplish what the agent prompted
-- Speed and efficiency: did Codebuff make unnecessary changes or take unnecessary steps? The task took ${durationSeconds} seconds - was this reasonable for the complexity?
 - Error: If there was an error encountered, you should give a very low score.
 
 Provide your response in a structured format with analysis, lists of strengths and weaknesses, and metrics.`

evals/git-evals/post-eval-analysis.ts

@@ -52,7 +52,6 @@ function buildAnalysisPrompt(evalResult: FullEvalLog): string {
   const metricsSection = `
 Overall Performance Metrics:
 - Average Completion Score: ${metrics.average_completion.toFixed(2)}/10
-- Average Efficiency Score: ${metrics.average_efficiency.toFixed(2)}/10
 - Average Code Quality Score: ${metrics.average_code_quality.toFixed(2)}/10
 - Average Overall Score: ${metrics.average_overall.toFixed(2)}/10
 - Average Duration: ${(metrics.average_duration_ms / 1000).toFixed(1)} seconds
@@ -73,7 +72,6 @@ Error: ${run.error || 'None'}
 
 Scores:
 - Completion: ${judging.metrics.completionScore}/10
-- Efficiency: ${judging.metrics.efficiencyScore}/10
 - Code Quality: ${judging.metrics.codeQualityScore}/10
 - Overall: ${judging.metrics.overallScore}/10
 

evals/git-evals/run-eval-set.ts

@@ -278,9 +278,6 @@ async function runEvalSet(options: {
         console.log(
           `   Completion: ${metrics.average_completion.toFixed(2)}/10`,
         )
-        console.log(
-          `   Efficiency: ${metrics.average_efficiency.toFixed(2)}/10`,
-        )
         console.log(
           `   Code Quality: ${metrics.average_code_quality.toFixed(2)}/10`,
         )
@@ -368,7 +365,6 @@ async function runEvalSet(options: {
               numCases: evalResult?.overall_metrics?.total_runs,
               avgScore: evalResult?.overall_metrics?.average_overall,
               avgCompletion: evalResult?.overall_metrics?.average_completion,
-              avgEfficiency: evalResult?.overall_metrics?.average_efficiency,
               avgCodeQuality: evalResult?.overall_metrics?.average_code_quality,
               avgDuration: evalResult?.overall_metrics?.average_duration_ms,
               suite: resultWrapper.name,

evals/git-evals/run-git-evals.ts

@@ -254,7 +254,6 @@ Explain your reasoning in detail.`,
         weaknesses: ['Judging process encountered an error'],
         metrics: {
           completionScore: 0,
-          efficiencyScore: 0,
           codeQualityScore: 0,
           overallScore: 0,
         },
@@ -620,11 +619,6 @@ function calculateOverallMetrics(evalRuns: EvalRunJudged[]) {
         (sum, run) => sum + (run.judging_results.metrics.completionScore || 0),
         0,
       ) / evalRuns.length,
-    average_efficiency:
-      evalRuns.reduce(
-        (sum, run) => sum + (run.judging_results.metrics.efficiencyScore || 0),
-        0,
-      ) / evalRuns.length,
     average_code_quality:
       evalRuns.reduce(
         (sum, run) => sum + (run.judging_results.metrics.codeQualityScore || 0),

evals/git-evals/run-single-eval.ts

@@ -212,7 +212,6 @@ async function runSingleEvalTask(options: {
         const metrics = result.judging_results.metrics
         console.log(`  Overall Score: ${metrics.overallScore.toFixed(2)}/10`)
         console.log(`  Completion: ${metrics.completionScore.toFixed(2)}/10`)
-        console.log(`  Efficiency: ${metrics.efficiencyScore.toFixed(2)}/10`)
         console.log(`  Code Quality: ${metrics.codeQualityScore.toFixed(2)}/10`)
 
         if (result.judging_results.strengths.length > 0) {

evals/git-evals/types.ts

@@ -64,7 +64,6 @@ export interface FullEvalLog {
     average_runtime_sec: number
     average_cost_usd: number
     average_completion: number
-    average_efficiency: number
     average_code_quality: number
     average_overall: number
     average_duration_ms: number
@@ -95,7 +94,6 @@ export const JudgingAnalysisSchema = z.object({
   weaknesses: z.array(z.string()),
   metrics: z.object({
     completionScore: z.number().min(0).max(10),
-    efficiencyScore: z.number().min(0).max(10),
     codeQualityScore: z.number().min(0).max(10),
     overallScore: z.number().min(0).max(10),
   }),