Skip to content

Commit 85a0cae

Browse files
LoCoBench Botclaude
andcommitted
feat: add sourcegraph_only config with Dockerfiles and run scripts
Implements the "upper bound" MCP config where agents have NO local source code and must use Sourcegraph MCP for all code access. Phase 2: sourcegraph_only agent code (in evals repo, committed separately) Phase 3: 14 Dockerfile.sg_only files for write-only suites - 5 K8s Docs (target dir only, no repo clone) - 5 LinuxFLBench (minimal image, no kernel) - 4 Investigation (no repo clone, output-only) Phase 4: Verifier wrapper + build-suite run script - sgonly_verifier_wrapper.sh: restore/overlay for build tasks - sgonly_build_2config.sh: SWE-bench Pro, PyTorch, Enterprise - sgonly_writeonly_2config.sh: K8s Docs, LinuxFLBench, Investigation Also adds sourcegraph_only to eval_matrix.json supported_configs. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 73c5ad0 commit 85a0cae

File tree

13 files changed

+981
-0
lines changed

13 files changed

+981
-0
lines changed
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM golang:1.23-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
python3 \
10+
npm \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Claude Code CLI
14+
RUN npm install -g @anthropic-ai/claude-code
15+
16+
# NO repo clone — agent uses Sourcegraph MCP exclusively for code access.
17+
# Investigation tasks produce /logs/agent/investigation.md only.
18+
RUN git init && \
19+
git config user.email "agent@example.com" && \
20+
git config user.name "Agent"
21+
22+
# Create output directories
23+
RUN mkdir -p /logs/agent /logs/verifier /app
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM golang:1.23-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
python3 \
10+
npm \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Claude Code CLI
14+
RUN npm install -g @anthropic-ai/claude-code
15+
16+
# NO repo clone — agent uses Sourcegraph MCP exclusively for code access.
17+
# Investigation tasks produce /logs/agent/investigation.md only.
18+
RUN git init && \
19+
git config user.email "agent@example.com" && \
20+
git config user.name "Agent"
21+
22+
# Create output directories
23+
RUN mkdir -p /logs/agent /logs/verifier /app
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
FROM python:3.12-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
npm \
10+
&& rm -rf /var/lib/apt/lists/*
11+
12+
# Install Claude Code CLI
13+
RUN npm install -g @anthropic-ai/claude-code
14+
15+
# NO repo clone — agent uses Sourcegraph MCP exclusively for code access.
16+
# Investigation tasks produce /logs/agent/investigation.md only.
17+
RUN git init && \
18+
git config user.email "agent@example.com" && \
19+
git config user.name "Agent"
20+
21+
# Create output directories
22+
RUN mkdir -p /logs/agent /logs/verifier /app
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM golang:1.23-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
python3 \
10+
npm \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Claude Code CLI
14+
RUN npm install -g @anthropic-ai/claude-code
15+
16+
# NO repo clone — agent uses Sourcegraph MCP exclusively for code access.
17+
# Investigation tasks produce /logs/agent/investigation.md only.
18+
RUN git init && \
19+
git config user.email "agent@example.com" && \
20+
git config user.name "Agent"
21+
22+
# Create output directories
23+
RUN mkdir -p /logs/agent /logs/verifier /app
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM gcc:13-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies (no kernel clone needed — agent uses MCP for code access)
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
ripgrep \
10+
python3 \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Node.js and Claude Code CLI
14+
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
15+
apt-get install -y nodejs && \
16+
npm install -g @anthropic-ai/claude-code
17+
18+
# NO kernel source — agent reads kernel code via Sourcegraph MCP,
19+
# writes JSON answer locally.
20+
RUN mkdir -p /workspace /tests /logs /app && \
21+
git init && \
22+
git config user.email "agent@example.com" && \
23+
git config user.name "Agent"
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM gcc:13-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies (no kernel clone needed — agent uses MCP for code access)
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
ripgrep \
10+
python3 \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Node.js and Claude Code CLI
14+
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
15+
apt-get install -y nodejs && \
16+
npm install -g @anthropic-ai/claude-code
17+
18+
# NO kernel source — agent reads kernel code via Sourcegraph MCP,
19+
# writes JSON answer locally.
20+
RUN mkdir -p /workspace /tests /logs /app && \
21+
git init && \
22+
git config user.email "agent@example.com" && \
23+
git config user.name "Agent"
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM gcc:13-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies (no kernel clone needed — agent uses MCP for code access)
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
ripgrep \
10+
python3 \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Node.js and Claude Code CLI
14+
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
15+
apt-get install -y nodejs && \
16+
npm install -g @anthropic-ai/claude-code
17+
18+
# NO kernel source — agent reads kernel code via Sourcegraph MCP,
19+
# writes JSON answer locally.
20+
RUN mkdir -p /workspace /tests /logs /app && \
21+
git init && \
22+
git config user.email "agent@example.com" && \
23+
git config user.name "Agent"
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM gcc:13-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies (no kernel clone needed — agent uses MCP for code access)
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
ripgrep \
10+
python3 \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Node.js and Claude Code CLI
14+
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
15+
apt-get install -y nodejs && \
16+
npm install -g @anthropic-ai/claude-code
17+
18+
# NO kernel source — agent reads kernel code via Sourcegraph MCP,
19+
# writes JSON answer locally.
20+
RUN mkdir -p /workspace /tests /logs /app && \
21+
git init && \
22+
git config user.email "agent@example.com" && \
23+
git config user.name "Agent"
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM gcc:13-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies (no kernel clone needed — agent uses MCP for code access)
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
ripgrep \
10+
python3 \
11+
&& rm -rf /var/lib/apt/lists/*
12+
13+
# Install Node.js and Claude Code CLI
14+
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
15+
apt-get install -y nodejs && \
16+
npm install -g @anthropic-ai/claude-code
17+
18+
# NO kernel source — agent reads kernel code via Sourcegraph MCP,
19+
# writes JSON answer locally.
20+
RUN mkdir -p /workspace /tests /logs /app && \
21+
git init && \
22+
git config user.email "agent@example.com" && \
23+
git config user.name "Agent"

configs/eval_matrix.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"sourcegraph_base",
1010
"sourcegraph_full",
1111
"sourcegraph_isolated",
12+
"sourcegraph_only",
1213
"github_base",
1314
"github_full"
1415
],

0 commit comments

Comments
 (0)