Skip to content

Commit dd1111c

Browse files
brandonkachenbrandonkachen
authored
fix(evals): temp fix for import paths (#172)
1 parent e2fa602 commit dd1111c

File tree

8 files changed

+47
-36
lines changed

8 files changed

+47
-36
lines changed

‎bun.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎evals/git-evals/gen-evals.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import path from 'path'
44
import { chunk } from 'lodash'
55
import { z } from 'zod'
66

7-
import { claudeModels, geminiModels } from 'common/src/constants'
8-
import { promptAiSdkStructured } from 'backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
7+
import { claudeModels, geminiModels } from '../../common/src/constants'
8+
import { promptAiSdkStructured } from '../../backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
99
import {
1010
CommitInfo,
1111
CommitSelectionSchema,

‎evals/git-evals/judge-git-eval.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import { promptAiSdkStructured } from 'backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
2-
import { geminiModels } from 'common/constants'
3-
import { generateCompactId } from 'common/util/string'
4-
import { countTokens } from 'backend/src/util/token-counter'
1+
import { promptAiSdkStructured } from '../../backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
2+
import { geminiModels } from '../../common/src/constants'
3+
import { generateCompactId } from '../../common/src/util/string'
4+
import { countTokens } from '../../backend/src/util/token-counter'
55
import { EvalRunLog, JudgingAnalysisSchema } from './types'
66
import { createPatch } from 'diff'
77

‎evals/git-evals/post-eval-analysis.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import { promptAiSdkStructured } from 'backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
2-
import { countTokens } from 'backend/src/util/token-counter'
3-
import { geminiModels } from 'common/constants'
4-
import { generateCompactId } from 'common/util/string'
1+
import { promptAiSdkStructured } from '../../backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
2+
import { countTokens } from '../../backend/src/util/token-counter'
3+
import { geminiModels } from '../../common/src/constants'
4+
import { generateCompactId } from '../../common/src/util/string'
55
import { z } from 'zod'
66
import { FullEvalLog } from './types'
77

@@ -51,7 +51,7 @@ function buildAnalysisPrompt(evalResult: FullEvalLog): string {
5151
const metricsSection = `
5252
Overall Performance Metrics:
5353
- Average Completion Score: ${metrics.average_completion.toFixed(2)}/10
54-
- Average Efficiency Score: ${metrics.average_efficiency.toFixed(2)}/10
54+
- Average Efficiency Score: ${metrics.average_efficiency.toFixed(2)}/10
5555
- Average Code Quality Score: ${metrics.average_code_quality.toFixed(2)}/10
5656
- Average Overall Score: ${metrics.average_overall.toFixed(2)}/10
5757
- Average Duration: ${(metrics.average_duration_ms / 1000).toFixed(1)} seconds

‎evals/git-evals/run-git-evals.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ import { execSync } from 'child_process'
22
import fs from 'fs'
33
import path from 'path'
44

5-
import { promptAiSdkStructured } from 'backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
6-
import { claudeModels } from 'common/src/constants'
7-
import { withTimeout } from 'common/util/promise'
8-
import { generateCompactId } from 'common/util/string'
9-
import { setProjectRoot, setWorkingDirectory } from 'npm-app/project-files'
10-
import { recreateShell } from 'npm-app/terminal/base'
5+
import { promptAiSdkStructured } from '../../backend/src/llm-apis/vercel-ai-sdk/ai-sdk'
6+
import { claudeModels } from '../../common/src/constants'
7+
import { withTimeout } from '../../common/src/util/promise'
8+
import { generateCompactId } from '../../common/src/util/string'
9+
import { setProjectRoot, setWorkingDirectory } from '../../npm-app/src/project-files'
10+
import { recreateShell } from '../../npm-app/src/terminal/base'
1111
import {
1212
createFileReadingMock,
1313
loopMainPrompt,

‎evals/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
"test:e2e-cat-app": "bun run e2e-cat-app-script.ts",
1111
"typecheck-this-package": "tsc --noEmit",
1212
"typecheck": "tsc --noEmit",
13-
"gen-git-evals": "bun run git-evals/gen-evals.ts",
14-
"run-git-evals": "bun run git-evals/run-git-evals.ts",
15-
"run-eval-set": "bun run git-evals/run-eval-set.ts",
13+
"gen-git-evals": "if command -v infisical >/dev/null 2>&1; then infisical run -- bun run git-evals/gen-evals.ts; else bun run git-evals/gen-evals.ts; fi",
14+
"run-git-evals": "if command -v infisical >/dev/null 2>&1; then infisical run -- bun run git-evals/run-git-evals.ts; else bun run git-evals/run-git-evals.ts; fi",
15+
"run-eval-set": "if command -v infisical >/dev/null 2>&1; then infisical run -- bun run git-evals/run-eval-set.ts; else bun run git-evals/run-eval-set.ts; fi",
1616
"setup-codebuff-repo": "bun run setup-codebuff-repo.ts"
1717
},
1818
"dependencies": {

‎evals/scaffolding.ts‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,22 @@ import { EventEmitter } from 'events'
33
import fs from 'fs'
44
import path from 'path'
55

6-
import * as mainPromptModule from 'backend/main-prompt'
7-
import { ClientToolCall } from 'backend/tools'
6+
import * as mainPromptModule from '../backend/src/main-prompt'
7+
import { ClientToolCall } from '../backend/src/tools'
88
import { mock } from 'bun:test'
9-
import { getFileTokenScores } from 'code-map/parse'
10-
import { FileChanges } from 'common/actions'
11-
import { TEST_USER_ID } from 'common/constants'
9+
import { getFileTokenScores } from '../packages/code-map/parse'
10+
import { FileChanges } from '../common/src/actions'
11+
import { TEST_USER_ID } from '../common/src/constants'
1212
import {
1313
getAllFilePaths,
1414
getProjectFileTree,
15-
} from 'common/src/project-file-tree'
16-
import { AgentState, ToolResult } from 'common/src/types/agent-state'
17-
import { applyAndRevertChanges } from 'common/util/changes'
18-
import { ProjectFileContext } from 'common/util/file'
19-
import { generateCompactId } from 'common/util/string'
20-
import { handleToolCall } from 'npm-app/tool-handlers'
21-
import { getSystemInfo } from 'npm-app/utils/system-info'
15+
} from '../common/src/project-file-tree'
16+
import { AgentState, ToolResult } from '../common/src/types/agent-state'
17+
import { applyAndRevertChanges } from '../common/src/util/changes'
18+
import { ProjectFileContext } from '../common/src/util/file'
19+
import { generateCompactId } from '../common/src/util/string'
20+
import { handleToolCall } from '../npm-app/src/tool-handlers'
21+
import { getSystemInfo } from '../npm-app/src/utils/system-info'
2222
import { blue } from 'picocolors'
2323
import { WebSocket } from 'ws'
2424

@@ -285,3 +285,14 @@ export function resetRepoToCommit(projectPath: string, commit: string) {
285285
throw error
286286
}
287287
}
288+
289+
export default {
290+
createFileReadingMock,
291+
getProjectFileContext,
292+
runMainPrompt,
293+
runToolCalls,
294+
loopMainPrompt,
295+
extractErrorFiles,
296+
applyAndRevertChangesSequentially,
297+
resetRepoToCommit,
298+
}

‎evals/test-setup.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import { execSync } from 'child_process'
22
import fs from 'fs'
33
import path from 'path'
44

5-
import { getInitialAgentState } from 'common/types/agent-state'
6-
import { setProjectRoot, setWorkingDirectory } from 'npm-app/project-files'
7-
import { recreateShell } from 'npm-app/terminal/base'
5+
import { getInitialAgentState } from '../common/src/types/agent-state'
6+
import { setProjectRoot, setWorkingDirectory } from '../npm-app/src/project-files'
7+
import { recreateShell } from '../npm-app/src/terminal/base'
88

99
import {
1010
createFileReadingMock,

0 commit comments

Comments
 (0)