Skip to content

Commit 4035bab

Browse files
pablogsalpablogsal
committed
Fix worker isolation problems
1 parent 5966ed4 commit 4035bab

File tree

1 file changed

+95
-40
lines changed

1 file changed

+95
-40
lines changed

‎worker/src/memory_tracker_worker/processing.py‎

Lines changed: 95 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -28,42 +28,10 @@ def process_commits(
2828
try:
2929
repo = git.Repo(repo_path)
3030

31-
# Clean the repository first
32-
logger.info("Cleaning repository with git clean -fxd")
33-
repo.git.clean("-fxd")
34-
35-
# Configure once at the beginning
36-
logger.info("Running configure once for local checkout mode")
37-
logger.debug(f"Configure flags: {configure_flags}")
38-
39-
configure_cmd = [str(repo_path / "configure"), *configure_flags.split()]
40-
logger.debug(f"Configure command: {' '.join(configure_cmd)}")
41-
42-
result = subprocess.run(
43-
configure_cmd, cwd=repo_path, check=True, capture_output=verbose < 3
44-
)
45-
if verbose >= 3:
46-
if result.stdout:
47-
print(
48-
result.stdout.decode()
49-
if isinstance(result.stdout, bytes)
50-
else result.stdout
51-
)
52-
if result.stderr:
53-
print(
54-
result.stderr.decode()
55-
if isinstance(result.stderr, bytes)
56-
else result.stderr
57-
)
58-
5931
# Process each commit
6032
for commit in commits:
6133
logger.info(f"Processing commit {commit.hexsha[:8]} in local checkout mode")
6234

63-
# Checkout the commit
64-
logger.info(f"Checking out commit {commit.hexsha[:8]}")
65-
repo.git.checkout(commit.hexsha)
66-
6735
# Create unique directory for this run
6836
run_dir = output_dir / commit.hexsha
6937

@@ -90,6 +58,69 @@ def process_commits(
9058
logger.debug(f"Created run directory: {run_dir}")
9159

9260
try:
61+
# Create parent temp directory for this commit
62+
parent_temp_dir = Path(tempfile.mkdtemp(prefix="cpython_build_"))
63+
logger.debug(f"Parent temp directory: {parent_temp_dir}")
64+
65+
# Clone CPython repo into temp directory
66+
cpython_repo_dir = parent_temp_dir / "cpython"
67+
logger.info(f"Cloning CPython repo to temp directory for commit {commit.hexsha[:8]}")
68+
cloned_repo = git.Repo.clone_from(str(repo_path), str(cpython_repo_dir))
69+
cloned_repo.git.checkout(commit.hexsha)
70+
logger.debug(f"CPython cloned to: {cpython_repo_dir}")
71+
72+
# Create install directory within parent temp dir
73+
install_dir = parent_temp_dir / "install"
74+
install_dir.mkdir(parents=True, exist_ok=True)
75+
logger.debug(f"Install directory: {install_dir}")
76+
77+
# Configure for this commit with prefix
78+
logger.info(f"Running configure for commit {commit.hexsha[:8]}")
79+
logger.debug(f"Configure flags: {configure_flags}")
80+
81+
configure_cmd = [str(cpython_repo_dir / "configure"), f"--prefix={install_dir}", *configure_flags.split()]
82+
logger.debug(f"Configure command: {' '.join(configure_cmd)}")
83+
84+
result = subprocess.run(
85+
configure_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3
86+
)
87+
if verbose >= 3:
88+
if result.stdout:
89+
print(
90+
result.stdout.decode()
91+
if isinstance(result.stdout, bytes)
92+
else result.stdout
93+
)
94+
if result.stderr:
95+
print(
96+
result.stderr.decode()
97+
if isinstance(result.stderr, bytes)
98+
else result.stderr
99+
)
100+
101+
# Clean before building
102+
logger.info(f"Running make clean for commit {commit.hexsha[:8]}")
103+
104+
clean_cmd = ["make", "clean"]
105+
logger.debug(f"Clean command: {' '.join(clean_cmd)}")
106+
107+
result = subprocess.run(
108+
clean_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3
109+
)
110+
if verbose >= 3:
111+
if result.stdout:
112+
print(
113+
result.stdout.decode()
114+
if isinstance(result.stdout, bytes)
115+
else result.stdout
116+
)
117+
if result.stderr:
118+
print(
119+
result.stderr.decode()
120+
if isinstance(result.stderr, bytes)
121+
else result.stderr
122+
)
123+
93124
# Build Python using make (no make install)
94125
logger.info(f"Running make for commit {commit.hexsha[:8]}")
95126
logger.debug(f"Make flags: {make_flags}")
@@ -98,7 +129,30 @@ def process_commits(
98129
logger.debug(f"Make command: {' '.join(make_cmd)}")
99130

100131
result = subprocess.run(
101-
make_cmd, cwd=repo_path, check=True, capture_output=verbose < 3
132+
make_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3
133+
)
134+
if verbose >= 3:
135+
if result.stdout:
136+
print(
137+
result.stdout.decode()
138+
if isinstance(result.stdout, bytes)
139+
else result.stdout
140+
)
141+
if result.stderr:
142+
print(
143+
result.stderr.decode()
144+
if isinstance(result.stderr, bytes)
145+
else result.stderr
146+
)
147+
148+
# Install Python
149+
logger.info(f"Running make install for commit {commit.hexsha[:8]}")
150+
151+
install_cmd = ["make", "install"]
152+
logger.debug(f"Install command: {' '.join(install_cmd)}")
153+
154+
result = subprocess.run(
155+
install_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3
102156
)
103157
if verbose >= 3:
104158
if result.stdout:
@@ -114,19 +168,19 @@ def process_commits(
114168
else result.stderr
115169
)
116170

117-
# Create virtual environment using local python binary
171+
# Create virtual environment using installed python binary
118172
logger.info(
119173
f"Creating virtual environment for commit {commit.hexsha[:8]}"
120174
)
121-
venv_dir = Path(tempfile.mkdtemp(prefix="cpython_venv_"))
175+
venv_dir = parent_temp_dir / "venv"
122176
logger.debug(f"Creating virtual environment in {venv_dir}")
123177

124-
python_binary = repo_path / "python"
178+
python_binary = install_dir / "bin" / "python3"
125179
venv_cmd = [str(python_binary), "-m", "venv", str(venv_dir)]
126180
logger.debug(f"Venv command: {' '.join(venv_cmd)}")
127181

128182
result = subprocess.run(
129-
venv_cmd, check=True, capture_output=verbose < 3
183+
venv_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3
130184
)
131185
if verbose >= 3:
132186
if result.stdout:
@@ -209,11 +263,12 @@ def process_commits(
209263
f"Successfully completed processing commit {commit.hexsha[:8]}"
210264
)
211265

212-
# Clean up venv directory
266+
# Clean up parent temp directory
213267
try:
214-
shutil.rmtree(venv_dir, ignore_errors=True)
268+
shutil.rmtree(parent_temp_dir, ignore_errors=True)
269+
logger.debug(f"Cleaned up parent temp directory: {parent_temp_dir}")
215270
except Exception as e:
216-
logger.warning(f"Failed to clean up venv directory {venv_dir}: {e}")
271+
logger.warning(f"Failed to clean up parent temp directory {parent_temp_dir}: {e}")
217272

218273
except subprocess.CalledProcessError as e:
219274
error_msg = f"Error processing commit {commit.hexsha}: {e}"

0 commit comments

Comments
 (0)