Skip to content

Commit 1e170cd

Browse files
nickatundonickatundo
committed
[#3] Download dev headers if not present
1 parent 0d4abb0 commit 1e170cd

3 files changed

Lines changed: 235 additions & 36 deletions

File tree

‎python/setup.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
# setup.py
2+
import os
3+
24
from setuptools import Extension, setup
35

6+
try:
7+
include_dirs = os.environ["PYTHON_INCLUDE_DIRS"].split(os.pathsep)
8+
except KeyError:
9+
raise RuntimeError(
10+
"PYTHON_INCLUDE_DIRS environment variable not set."
11+
)
12+
413
module = Extension(
514
"ubeacon",
615
sources=[
@@ -10,6 +19,7 @@
1019
"src/ext/cJSON/cJSON.c",
1120
],
1221
extra_compile_args=["-O0", "-Isrc/ext/cJSON", "-std=c99"],
22+
include_dirs=include_dirs,
1323
)
1424

1525
setup(

‎python/src/ubeacon/extension/download_python_headers.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
"""
2+
Download Python development headers for a given version into a temporary directory.
3+
4+
Works without root on Ubuntu, RHEL, Fedora, and OpenSUSE by downloading (not installing)
5+
the appropriate package and extracting it locally.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import subprocess
11+
from pathlib import Path
12+
13+
import distro
14+
15+
16+
def _run_command(command: list[str], **kwargs: object) -> subprocess.CompletedProcess:
17+
"""
18+
Run a command with subprocess.run and check for errors.
19+
20+
command: The command to run, as a list of arguments.
21+
kwargs: Additional keyword arguments to pass to subprocess.run.
22+
23+
Returns the CompletedProcess object returned by subprocess.run.
24+
"""
25+
return subprocess.run(command, check=True, capture_output=True, text=True, **kwargs)
26+
27+
28+
def _get_download_package_uri(version: str) -> str:
29+
"""
30+
Return the URL of a distribution package file.
31+
"""
32+
33+
match distro.id():
34+
case dist if dist in {"ubuntu", "debian"}:
35+
36+
return (
37+
_run_command(
38+
["apt-get", "--print-uris", "download", f"libpython{version}-dev"]
39+
)
40+
.stdout.splitlines()[0]
41+
.split()[0]
42+
.strip("'")
43+
)
44+
45+
case dist if dist in {"fedora", "rhel", "rocky", "centos", "amzn"}:
46+
47+
def dnf(package_name: str) -> str:
48+
# TODO support on ARM
49+
return _run_command(
50+
[
51+
"dnf",
52+
"repoquery",
53+
"--location",
54+
package_name,
55+
"--archlist",
56+
"x86_64",
57+
]
58+
).stdout.strip()
59+
60+
rpms = dnf(f"python{version}-devel")
61+
if not rpms:
62+
# The package may be named python3-devel if it's the default
63+
# Python version for the current distro version.
64+
rpms = dnf("python3-devel")
65+
if version not in rpms:
66+
raise ValueError(
67+
f"Could not find a suitable python-devel package for version {version} in dnf output: {rpms}"
68+
)
69+
return rpms.splitlines()[
70+
0
71+
] # Take the first result if there are multiple matches.
72+
73+
case _:
74+
raise ValueError(f"Unsupported distribution: {distro.id()}")
75+
76+
77+
def _extract_deb(package_path: Path, extract_dir: Path) -> None:
78+
"""Extract a .deb package into the given directory."""
79+
_run_command(["dpkg-deb", "-x", str(package_path), str(extract_dir)])
80+
81+
82+
def _extract_rpm(package_path: Path, extract_dir: Path) -> None:
83+
"""Extract a .rpm package into the given directory."""
84+
with subprocess.Popen(
85+
["rpm2cpio", str(package_path)], stdout=subprocess.PIPE
86+
) as rpm2cpio:
87+
_run_command(["cpio", "-idm"], stdin=rpm2cpio.stdout, cwd=extract_dir)
88+
rpm2cpio.wait()
89+
if rpm2cpio.returncode != 0:
90+
raise subprocess.CalledProcessError(
91+
rpm2cpio.returncode, ["rpm2cpio", str(package_path)]
92+
)
93+
94+
95+
def python_dev_headers(
96+
version: str, storage_dir: Path, uri_override: str | None = None
97+
) -> Path:
98+
"""
99+
Download the appropriate python-dev/python-devel package for the current Linux
100+
distribution if necessary, extract it, and return the path to the
101+
extracted headers.
102+
103+
Args:
104+
version: The Python version string, e.g. "3.11".
105+
storage_dir: The directory to use for downloading and extracting packages.
106+
uri_override: If provided, this URI will be used instead of determining
107+
the package URL based on the distribution. This is intended for testing.
108+
109+
Returns:
110+
The root of the unpacked headers.
111+
112+
Raises:
113+
ValueError: If the current distribution is not supported.
114+
subprocess.CalledProcessError: If downloading or extracting the package fails.
115+
"""
116+
download_dir = storage_dir / "packages"
117+
download_dir.mkdir(exist_ok=True)
118+
119+
uri = uri_override or _get_download_package_uri(version)
120+
121+
# Fetch the package file to the download directory.
122+
package_path = download_dir / uri.split("/")[-1]
123+
if not package_path.exists():
124+
_run_command(["curl", "--fail","-L", "-o", str(package_path), uri])
125+
126+
extract_dir = storage_dir / (package_path.stem + "_extracted")
127+
if not extract_dir.exists():
128+
if package_path.suffix == ".deb":
129+
extract = _extract_deb
130+
elif package_path.suffix == ".rpm":
131+
extract = _extract_rpm
132+
else:
133+
raise ValueError(
134+
f"""Unknown package format: {package_path.suffix}. Expected .deb or .rpm."""
135+
)
136+
137+
extract_dir.mkdir(exist_ok=True)
138+
extract(package_path, extract_dir)
139+
140+
return extract_dir

‎python/src/ubeacon/extension/ubeacon.py‎

Lines changed: 85 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import functools
1313
import json
1414
import os
15+
import re
1516
import subprocess
1617
import tempfile
1718
from pathlib import Path
@@ -23,9 +24,11 @@
2324
import pygments.formatters
2425
import pygments.lexers
2526
from src.udbpy import locations, report # pyright: ignore[reportMissingModuleSource]
26-
from src.udbpy.gdb_extensions import gdbutils # pyright: ignore[reportMissingModuleSource]
27+
from src.udbpy.gdb_extensions import (
28+
gdbutils,
29+
) # pyright: ignore[reportMissingModuleSource]
2730

28-
from . import debuggee, messages
31+
from . import debuggee, download_python_headers, messages
2932

3033
PREFIX = "s_ubeacon"
3134
STATE_STRUCT = PREFIX
@@ -36,6 +39,84 @@
3639
EXCEPTION_FN = f"{TRACE_PREFIX}_exception"
3740

3841

42+
def wrap_build(python_executable: str, cache_dir: Path, addon_root: Path) -> None:
43+
"""Build the UBeacon library using the specified Python executable.
44+
45+
If the Python development headers are not available on the system, they are
46+
downloaded using the distro's package manager.
47+
48+
python_executable:
49+
The path to the Python executable to build against.
50+
cache_dir:
51+
The directory to use for caching build artifacts and downloaded headers.
52+
addon_root:
53+
The root directory of the addon, where `setup.py` is located.
54+
"""
55+
56+
version_string = subprocess.check_output(
57+
[
58+
python_executable,
59+
"-c",
60+
"import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')",
61+
],
62+
text=True,
63+
cwd=addon_root,
64+
stderr=subprocess.STDOUT,
65+
).strip()
66+
67+
# Does this Python executable have development headers installed already?
68+
include_paths = subprocess.check_output(
69+
[
70+
python_executable,
71+
"-c",
72+
"import sysconfig; print(sysconfig.get_path('include'))",
73+
],
74+
text=True,
75+
).strip()
76+
77+
if not Path(include_paths).exists():
78+
# Fetch headers from a distro package
79+
header_dir = download_python_headers.python_dev_headers(
80+
version_string, storage_dir=cache_dir
81+
)
82+
include_paths = f"{header_dir}/usr/include:{header_dir}/usr/include/python{version_string}"
83+
84+
try:
85+
subprocess.run(
86+
[
87+
python_executable,
88+
"setup.py",
89+
"build",
90+
"--quiet",
91+
f"--build-base={cache_dir}",
92+
],
93+
text=True,
94+
cwd=addon_root,
95+
check=True,
96+
capture_output=True,
97+
env={**os.environ, "PYTHON_INCLUDE_DIRS": include_paths},)
98+
99+
except subprocess.CalledProcessError as exc:
100+
output = ""
101+
if exc.output:
102+
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
103+
tf.write(exc.output)
104+
tf.flush()
105+
output += f"Saved stdout to {tf.name}.\n"
106+
if exc.stderr:
107+
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
108+
tf.write(exc.stderr)
109+
tf.flush()
110+
output += f"Saved stderr to {tf.name}.\n"
111+
raise report.ReportableError(
112+
f"""Error occurred in Python: could not debug this version of Python.
113+
114+
You may need to install Python development headers for this version.
115+
116+
{output}"""
117+
)
118+
119+
39120
@functools.cache
40121
def build() -> Path:
41122
"""
@@ -81,40 +162,8 @@ def build() -> Path:
81162
return lib_path
82163

83164
# Not found, build it
84-
try:
85-
subprocess.run(
86-
[
87-
python_executable,
88-
"setup.py",
89-
"build",
90-
"--quiet",
91-
f"--build-base={cache_dir}",
92-
],
93-
text=True,
94-
cwd=root,
95-
check=True,
96-
capture_output=True,
97-
)
98-
except subprocess.CalledProcessError as exc:
99-
if exc.output:
100-
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
101-
tf.write(exc.output)
102-
tf.flush()
103-
report.user(
104-
f"Saved stdout to {tf.name}.\n"
105-
)
106-
if exc.stderr:
107-
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
108-
tf.write(exc.stderr)
109-
tf.flush()
110-
report.user(
111-
f"Saved stderr to {tf.name}.\n"
112-
)
113-
raise report.ReportableError(
114-
"""Error occurred in Python: could not debug this version of Python.
115-
116-
You may need to install Python development headers for this version."""
117-
)
165+
wrap_build(python_executable, cache_dir, root)
166+
118167
output = subprocess.check_output(
119168
[python_executable, "find_so.py", cache_dir], text=True, cwd=root
120169
)

0 commit comments

Comments
 (0)