[feat] benchify integration (#312)

Co-authored-by: Cole Vick <cole.d.vick@gmail.com>

Author: brandonkachen

.github/workflows/npm-app-release-staging.yml

@@ -134,7 +134,7 @@ jobs:
       new-version: ${{ needs.prepare-and-commit-staging.outputs.new_version }}
       artifact-name: updated-staging-package
       checkout-ref: ${{ github.event.pull_request.head.sha }}
-      env-overrides: '{"NEXT_PUBLIC_CB_ENVIRONMENT": "prod", "NEXT_PUBLIC_CODEBUFF_BACKEND_URL": "backend-pr-221-we0m.onrender.com"}'
+      env-overrides: '{"NEXT_PUBLIC_CB_ENVIRONMENT": "prod", "NEXT_PUBLIC_CODEBUFF_BACKEND_URL": "backend-pr-312-3hui.onrender.com"}'
     secrets: inherit
 
   # Create GitHub prerelease with all binaries

backend/package.json

@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@ai-sdk/google-vertex": "3.0.6",
+    "benchify": "^0.1.0-alpha.41",
     "@ai-sdk/openai": "2.0.11",
     "@codebuff/billing": "workspace:*",
     "@codebuff/common": "workspace:*",

backend/src/__tests__/process-str-replace.test.ts

@@ -1,7 +1,22 @@
-import { describe, expect, it } from 'bun:test'
+import { describe, expect, it, spyOn, beforeEach, afterEach, mock } from 'bun:test'
 import { applyPatch } from 'diff'
 
+// Mock the benchify module to simulate missing API key
+mock.module('benchify', () => ({
+  Benchify: class MockBenchify {
+    constructor() {}
+    runFixer() {
+      return Promise.resolve([])
+    }
+  }
+}))
+
 import { processStrReplace } from '../process-str-replace'
+import { mockFileContext } from './test-utils'
+import {
+  executeBatchStrReplaces,
+  benchifyCanFixLanguage,
+} from '../tools/batch-str-replace'
 
 describe('processStrReplace', () => {
   it('should replace exact string matches', async () => {
@@ -213,6 +228,25 @@ describe('processStrReplace', () => {
     }
   })
 
+  it('should handle replacement where old string equals new string', async () => {
+    const initialContent = 'const x = 1;\nconst y = 2;\n'
+    const oldStr = 'const y = 2;'
+    const newStr = 'const y = 2;' // Same as old string
+
+    const result = await processStrReplace(
+      'test.ts',
+      [{ old: oldStr, new: newStr, allowMultiple: false }],
+      Promise.resolve(initialContent),
+    )
+
+    expect(result).not.toBeNull()
+    expect('content' in result).toBe(true)
+    if ('content' in result) {
+      expect(result.content).toBe('const x = 1;\nconst y = 2;\n')
+      expect(result.messages).toEqual([])
+    }
+  })
+
   // New comprehensive tests for allowMultiple functionality
   describe('allowMultiple functionality', () => {
     it('should error when multiple occurrences exist and allowMultiple is false', async () => {
@@ -417,3 +451,150 @@ function test3() {
     )
   })
 })
+
+// Tests for Benchify resilience
+describe('Benchify resilience', () => {
+  describe('happy path', () => {
+    it('should identify Benchify-supported file types correctly', () => {
+      const testCases = [
+        { path: 'component.tsx', expected: true },
+        { path: 'utils.ts', expected: true },
+        { path: 'script.js', expected: true },
+        { path: 'styles.jsx', expected: true },
+        { path: 'README.md', expected: false },
+        { path: 'config.json', expected: false },
+        { path: 'styles.css', expected: false },
+        { path: 'index.html', expected: false },
+        { path: 'test.py', expected: false },
+      ]
+
+      for (const { path, expected } of testCases) {
+        const result = benchifyCanFixLanguage(path)
+        expect(result).toBe(expected)
+      }
+    })
+
+    it('should handle file extensions case sensitivity', () => {
+      expect(benchifyCanFixLanguage('Component.TSX')).toBe(false) // Wrong case
+      expect(benchifyCanFixLanguage('component.tsx')).toBe(true) // Correct case
+      expect(benchifyCanFixLanguage('utils.TS')).toBe(false) // Wrong case
+      expect(benchifyCanFixLanguage('utils.ts')).toBe(true) // Correct case
+    })
+
+    it('should handle file paths with multiple dots', () => {
+      expect(benchifyCanFixLanguage('component.test.tsx')).toBe(true)
+      expect(benchifyCanFixLanguage('utils.spec.ts')).toBe(true)
+      expect(benchifyCanFixLanguage('config.local.js')).toBe(true)
+      expect(benchifyCanFixLanguage('styles.module.css')).toBe(false)
+    })
+
+    it('should handle files without extensions', () => {
+      expect(benchifyCanFixLanguage('Dockerfile')).toBe(false)
+      expect(benchifyCanFixLanguage('Makefile')).toBe(false)
+      expect(benchifyCanFixLanguage('README')).toBe(false)
+    })
+  })
+
+  it('should fall back gracefully when Benchify is disabled', async () => {
+    // Mock the process.env to simulate missing BENCHIFY_API_KEY
+    const originalEnv = process.env.BENCHIFY_API_KEY
+    delete process.env.BENCHIFY_API_KEY
+
+    try {
+      const result = await executeBatchStrReplaces({
+        deferredStrReplaces: [
+          {
+            toolCall: {
+              toolName: 'str_replace' as const,
+              toolCallId: 'test-call',
+              input: {
+                path: 'test.ts',
+                replacements: [
+                  { old: 'old', new: 'new', allowMultiple: false },
+                ],
+              },
+            },
+          },
+        ],
+        toolCalls: [],
+        toolResults: [],
+        ws: {} as any,
+        fileContext: mockFileContext,
+        agentStepId: 'test-step',
+        clientSessionId: 'test-session',
+        userInputId: 'test-input',
+        onResponseChunk: () => {},
+        state: { messages: [] },
+        userId: 'test-user',
+      })
+
+      // Should complete without error even when Benchify is unavailable
+      expect(result).toBeUndefined() // Function returns void
+    } finally {
+      // Restore the original environment variable
+      if (originalEnv !== undefined) {
+        process.env.BENCHIFY_API_KEY = originalEnv
+      }
+    }
+  })
+
+  describe('Batch str_replace integration tests', () => {
+    it('should handle empty deferred list without error', async () => {
+      // Simple test that doesn't require complex mocking
+      expect(
+        executeBatchStrReplaces({
+          deferredStrReplaces: [],
+          toolCalls: [],
+          toolResults: [],
+          ws: {} as any,
+          fileContext: mockFileContext,
+          agentStepId: 'test-step',
+          clientSessionId: 'test-session',
+          userInputId: 'test-input',
+          onResponseChunk: () => {},
+          state: { messages: [] },
+          userId: 'test-user',
+        }),
+      ).resolves.toBeUndefined() // Should complete without throwing
+    })
+  })
+
+  it('should identify Benchify-supported file types correctly', () => {
+    const testCases = [
+      { path: 'component.tsx', expected: true },
+      { path: 'utils.ts', expected: true },
+      { path: 'script.js', expected: true },
+      { path: 'styles.jsx', expected: true },
+      { path: 'README.md', expected: false },
+      { path: 'config.json', expected: false },
+      { path: 'styles.css', expected: false },
+      { path: 'index.html', expected: false },
+      { path: 'test.py', expected: false },
+    ]
+
+    for (const { path, expected } of testCases) {
+      const result = benchifyCanFixLanguage(path)
+      expect(result).toBe(expected)
+    }
+  })
+
+  it('should handle executeBatchStrReplaces with empty list', async () => {
+    // Simple test that doesn't require complex mocking
+    const result = await executeBatchStrReplaces({
+      deferredStrReplaces: [],
+      toolCalls: [],
+      toolResults: [],
+      ws: {} as any,
+      fileContext: mockFileContext,
+      agentStepId: 'test-step',
+      clientSessionId: 'test-session',
+      userInputId: 'test-input',
+      onResponseChunk: () => {},
+      state: { messages: [] },
+      userId: 'test-user',
+    })
+
+    // Should complete without throwing an error
+    expect(result).toBeUndefined() // Function returns void
+  })
+})

backend/src/process-str-replace.ts

@@ -35,6 +35,7 @@ export async function processStrReplace(
   let currentContent = initialContent
   let messages: string[] = []
   const lineEnding = currentContent.includes('\r\n') ? '\r\n' : '\n'
+  let anyReplacementSuccessful = false
 
   for (const { old: oldStr, new: newStr, allowMultiple } of replacements) {
     // Regular case: require oldStr for replacements
@@ -59,6 +60,7 @@ export async function processStrReplace(
 
     if (match.success) {
       updatedOldStr = match.oldStr
+      anyReplacementSuccessful = true
     } else {
       messages.push(match.error)
       updatedOldStr = null
@@ -72,15 +74,15 @@ export async function processStrReplace(
 
   currentContent = currentContent.replaceAll('\n', lineEnding)
 
-  if (initialContent === currentContent) {
+  // If no successful replacements occurred, return error
+  if (!anyReplacementSuccessful) {
     logger.debug(
       {
         path,
         initialContent,
       },
-      `processStrReplace: No change to ${path}`,
+      `processStrReplace: No successful replacements for ${path}`,
     )
-    messages.push('No change to the file.')
     return {
       tool: 'str_replace' as const,
       path,

backend/src/run-agent-step.ts

@@ -305,7 +305,6 @@ export const runAgentStep = async (
     state,
     fullResponse: fullResponseAfterStream,
     fullResponseChunks,
-    messageId,
   } = await processStreamWithTools({
     stream,
     ws,
@@ -414,7 +413,7 @@ export const runAgentStep = async (
     agentState,
     fullResponse,
     shouldEndTurn,
-    messageId,
+    messageId: null,
   }
 }