add cost breakdown to email

Author: charleslien

evals/git-evals/email-eval-results.ts

@@ -43,8 +43,25 @@ function formatEvalSummaryForEmail(
       (sum, result) => sum + result.overall_metrics.average_code_quality,
       0,
     ) / evalResults.length
+  const avgCostUsd =
+    evalResults.reduce(
+      (sum, result) => sum + result.overall_metrics.average_cost_usd,
+      0,
+    ) / evalResults.length
+  const totalCostUsd = evalResults.reduce(
+    (sum, result) =>
+      sum +
+      result.overall_metrics.average_cost_usd *
+        result.overall_metrics.total_runs,
+    0,
+  )
+  const avgRuntimeSec =
+    evalResults.reduce(
+      (sum, result) => sum + result.overall_metrics.average_runtime_sec,
+      0,
+    ) / evalResults.length
 
-  const subject = `Codebuf Eval Results - ${title ? title : new Date().toLocaleDateString()} - Overall Score: ${avgOverallScore.toFixed(1)}/10`
+  const subject = `Codebuff Eval Results - ${title ? title : new Date().toLocaleDateString()} - Score: ${avgOverallScore.toFixed(1)}/10 | Cost: ${avgCostUsd.toFixed(3)} | ${avgRuntimeSec.toFixed(1)}s`
 
   // Build the complete message as a single string
   const summary = analyses.map((analysis) => analysis.summary).join('\n\n')
@@ -56,13 +73,21 @@ function formatEvalSummaryForEmail(
 • Efficiency: ${avgEfficiency.toFixed(2)}/10
 • Code Quality: ${avgCodeQuality.toFixed(2)}/10
 
+💰 COST & PERFORMANCE METRICS
+• Average Cost per Run: ${avgCostUsd.toFixed(4)}
+• Total Cost: ${totalCostUsd.toFixed(2)}
+• Average Runtime: ${avgRuntimeSec.toFixed(1)} seconds
+• Cost per Point (Overall Score): ${(avgCostUsd / avgOverallScore).toFixed(4)}
+
 📈 BY EVAL SET:
 ${evalResults
   .map(
     (result) => `${result.test_repo_name}:
   - Success: ${result.overall_metrics.successful_runs}/${result.overall_metrics.total_runs}
   - Overall: ${result.overall_metrics.average_overall.toFixed(1)}/10
-  - Completion: ${result.overall_metrics.average_completion.toFixed(1)}/10`,
+  - Completion: ${result.overall_metrics.average_completion.toFixed(1)}/10
+  - Avg Cost: ${result.overall_metrics.average_cost_usd.toFixed(4)}
+  - Avg Runtime: ${result.overall_metrics.average_runtime_sec.toFixed(1)}s`,
   )
   .join('\n')}`
 
@@ -103,18 +128,40 @@ ${allProblems
   const recommendations = `💡 DEVELOPMENT RECOMMENDATIONS:
 ${uniqueRecommendations.map((rec, i) => `${i + 1}. ${rec}`).join('\n')}`
 
+  // Add detailed cost breakdown section
+  const costBreakdown = `💸 DETAILED COST BREAKDOWN
+${evalResults
+  .map((result) => {
+    const setCost =
+      result.overall_metrics.average_cost_usd *
+      result.overall_metrics.total_runs
+    const costPerSuccessfulRun =
+      result.overall_metrics.successful_runs > 0
+        ? setCost / result.overall_metrics.successful_runs
+        : 0
+    return `${result.test_repo_name}:
+  - Total Set Cost: ${setCost.toFixed(3)}
+  - Cost per Run: ${result.overall_metrics.average_cost_usd.toFixed(4)}
+  - Cost per Successful Run: ${costPerSuccessfulRun.toFixed(4)}
+  - Runtime Efficiency: ${result.overall_metrics.average_runtime_sec > 0 ? (result.overall_metrics.average_overall / result.overall_metrics.average_runtime_sec).toFixed(3) : 'N/A'} points/sec`
+  })
+  .join('\n')}`
+
   // Combine everything into a single message
   const message = `${summary}
 
 ${metrics}
 
+${costBreakdown}
+
 ${topProblems}
 
 ${recommendations}
 
 Generated on: ${new Date().toISOString()}
 Total Eval Sets: ${evalResults.length}
-Total Runs: ${totalRuns}`
+Total Runs: ${totalRuns}
+Total Budget Used: $${totalCostUsd.toFixed(2)}`
 
   return {
     subject,