Run stdlib tests

Cargo.toml

@@ -30,6 +30,7 @@ rustls = { version = "0.23" , default-features = false, features = ["aws_lc_rs"]
 semver = "1.0.27"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.145"
+serde_yaml = "0.9.33"
 sha2 = "0.10.9"
 tar = "0.4.44"
 tempfile = "3.23.0"

ci-runners.yaml

@@ -1,11 +1,11 @@
 # Describes the runners that the CI system can use
 
-depot-ubuntu-22.04-4:
+depot-ubuntu-22.04-32:
   arch: x86_64
   platform: linux
   free: false
 
-depot-ubuntu-22.04-arm-4:
+depot-ubuntu-22.04-arm-32:
   arch: aarch64
   platform: linux
   free: false

cpython-unix/build-cpython.sh

@@ -616,6 +616,11 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
     PROFILE_TASK="${PROFILE_TASK} --ignore test_json"
 fi
 
+# PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it.
+if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then
+    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
+fi
+
 # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,
 # but on x86-64 that requires v2 and v3 respectively, and on arm64 the performance is bad as noted
 # in the comments, so just don't even try. (We should check if we can make this conditional)
@@ -1315,7 +1320,7 @@ ${BUILD_PYTHON} "${ROOT}/fix_shebangs.py" "${ROOT}/out/python/install"
 # downstream consumers.
 OBJECT_DIRS="Objects Parser Parser/lexer Parser/pegen Parser/tokenizer Programs Python Python/deepfreeze"
 OBJECT_DIRS="${OBJECT_DIRS} Modules"
-for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _remote_debugging _sha3 _sqlite _sre _testinternalcapi _xxtestfuzz _zstd; do
+for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _remote_debugging _sha3 _sqlite _sre _testcapi _testinternalcapi _testlimitedcapi _xxtestfuzz _zstd; do
     OBJECT_DIRS="${OBJECT_DIRS} Modules/${ext}"
 done
 

cpython-unix/extension-modules.yml

@@ -607,32 +607,133 @@ _sysconfig:
 
 _testbuffer:
   minimum-python-version: '3.10'
+  build-mode: shared
   sources:
     - _testbuffer.c
 
-# _testcapi exists to test the public C APIs. It makes assumptions that it is
-# built as a shared library. Our static extension module builds invalidate this
-# assumption. So just disable globally.
 _testcapi:
-  disabled-targets:
-    - .*
+  # Must link against public API, which isn't available in static build.
+  build-mode: shared-or-disabled
   sources:
     - _testcapimodule.c
+  sources-conditional:
+    - source: _testcapi/abstract.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/buffer.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/bytearray.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/bytes.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/code.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/codec.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/complex.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/config.c
+      minimum-python-version: "3.14"
+    - source: _testcapi/datetime.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/dict.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/docstring.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/eval.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/exceptions.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/file.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/float.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/frame.c
+      minimum-python-version: "3.14"
+    - source: _testcapi/function.c
+      minimum-python-version: "3.14"
+    - source: _testcapi/gc.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/getargs.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/hash.c
+      minimum-python-version: "3.13"
+    - source: _testcapi/heaptype.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/heaptype_relative.c
+      maximum-python-version: "3.12"
+      minimum-python-version: "3.12"
+    - source: _testcapi/immortal.c
+      minimum-python-version: "3.12"
+    # import.c was added in 3.12, removed in 3.13, and added again in 3.14.
+    - source: _testcapi/import.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/import.c
+      minimum-python-version: "3.14"
+    - source: _testcapi/list.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/long.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/mem.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/modsupport.c
+      minimum-python-version: "3.15"
+    - source: _testcapi/module.c
+      minimum-python-version: "3.15"
+    - source: _testcapi/monitoring.c
+      minimum-python-version: "3.13"
+    - source: _testcapi/numbers.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/object.c
+      minimum-python-version: "3.13"
+    - source: _testcapi/pyatomic.c
+      minimum-python-version: "3.13"
+    - source: _testcapi/pyos.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/pytime.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/run.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/set.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/structmember.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/sys.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/time.c
+      minimum-python-version: "3.13"
+    - source: _testcapi/tuple.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/type.c
+      minimum-python-version: "3.14"
+    - source: _testcapi/unicode.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/vectorcall.c
+      minimum-python-version: "3.12"
+    - source: _testcapi/vectorcall_limited.c
+      minimum-python-version: "3.12"
+      maximum-python-version: "3.12"
+    - source: _testcapi/watchers.c
+      minimum-python-version: "3.12"
 
 _testexternalinspection:
   minimum-python-version: '3.13'
   maximum-python-version: '3.13'
+  build-mode: shared
   sources:
     - _testexternalinspection.c
 
 _testimportmultiple:
   minimum-python-version: '3.10'
+  build-mode: shared
   sources:
     - _testimportmultiple.c
 
-# Ideally we disable all test extensions. However, this one is used by a bunch
-# of tests, including tests that run during PGO profiling. We keep it enabled
-# so it doesn't break tests and undermine PGO.
 _testinternalcapi:
   includes:
     - Include/internal
@@ -657,13 +758,48 @@ _testinternalcapi:
     - source: _testinternalcapi/tuple.c
       minimum-python-version: "3.15"
 
+_testlimitedcapi:
+  minimum-python-version: "3.13"
+  # Must link against public API, which isn't available in static build.
+  build-mode: shared-or-disabled
+  sources:
+    - _testlimitedcapi.c
+    - _testlimitedcapi/abstract.c
+    - _testlimitedcapi/bytearray.c
+    - _testlimitedcapi/bytes.c
+    - _testlimitedcapi/complex.c
+    - _testlimitedcapi/dict.c
+    - _testlimitedcapi/eval.c
+    - _testlimitedcapi/file.c
+    - _testlimitedcapi/float.c
+    - _testlimitedcapi/heaptype_relative.c
+    - _testlimitedcapi/import.c
+    - _testlimitedcapi/list.c
+    - _testlimitedcapi/long.c
+    - _testlimitedcapi/object.c
+    - _testlimitedcapi/pyos.c
+    - _testlimitedcapi/set.c
+    - _testlimitedcapi/sys.c
+    - _testlimitedcapi/tuple.c
+    - _testlimitedcapi/unicode.c
+    - _testlimitedcapi/vectorcall_limited.c
+  sources-conditional:
+    - source: _testlimitedcapi/codec.c
+      minimum-python-version: "3.14"
+    - source: _testlimitedcapi/threadstate.c
+      minimum-python-version: "3.15"
+    - source: _testlimitedcapi/version.c
+      minimum-python-version: "3.14"
+
 _testmultiphase:
   minimum-python-version: '3.10'
+  build-mode: shared
   sources:
     - _testmultiphase.c
 
 _testsinglephase:
   minimum-python-version: '3.12'
+  build-mode: shared
   sources:
     - _testsinglephase.c
 
@@ -946,18 +1082,13 @@ unicodedata:
     - unicodedata.c
 
 xxlimited:
-  # Similar story as _testcapi. The extension exists to test the limited API,
-  # which we don't really care about. Statically building it runs into problems
-  # and cross-compiling emits wrong machine type when built via setup.py.
+  # Example extension. We don't care about it.
   disabled-targets:
     - .*
 
 xxlimited_35:
   minimum-python-version: '3.10'
-
-  # Similar story as _testcapi. The extension exists to test the limited API,
-  # which we don't really care about. Statically building it runs into problems
-  # and cross-compiling emits wrong machine type when built via setup.py.
+  # Example extension. We don't care about it.
   disabled-targets:
     - .*
 

pythonbuild/cpython.py

@@ -274,7 +274,12 @@ def derive_setup_local(
             python_version, info.get("maximum-python-version", "100.0")
         )
 
-        if info.get("build-mode") not in (None, "shared", "static"):
+        if info.get("build-mode") not in (
+            None,
+            "shared",
+            "static",
+            "shared-or-disabled",
+        ):
             raise Exception("unsupported build-mode for extension module %s" % name)
 
         if not (python_min_match and python_max_match):
@@ -290,6 +295,11 @@ def derive_setup_local(
                 )
                 disabled.add(name)
 
+        # Extension is demanding it be built as shared. If this isn't possible due to
+        # a static build, disable the extension.
+        if info.get("build-mode") == "shared-or-disabled" and "static" in build_options:
+            disabled.add(name)
+
         if info.get("setup-enabled", False):
             setup_enabled_wanted.add(name)
 
@@ -507,6 +517,11 @@ def derive_setup_local(
         section = (
             "static" if "static" in build_options else info.get("build-mode", "static")
         )
+
+        # shared-or-disabled maps to shared or disabled.
+        if section == "shared-or-disabled":
+            section = "shared"
+
         enabled_extensions[name]["build-mode"] = section
 
         # Presumably this means the extension comes from the distribution's

src/macho.rs

@@ -4,7 +4,7 @@
 
 use {
     crate::validation::ValidationContext,
-    anyhow::{Context, Result, anyhow},
+    anyhow::{anyhow, Context, Result},
     apple_sdk::{AppleSdk, SdkSearch, SdkSearchLocation, SdkSorting, SdkVersion, SimpleSdk},
     semver::Version,
     std::{

src/release.rs

@@ -448,6 +448,19 @@ pub fn build_wanted_filenames(
     Ok(wanted_filenames)
 }
 
+/// Extension modules that should not be included in "install only" archives.
+const INSTALL_ONLY_DROP_EXTENSIONS: &[&str] = &[
+    "_ctypes_test",
+    "_testbuffer",
+    "_testcapi",
+    "_testexternalinspection",
+    "_testimportmultiple",
+    "_testinternalcapi",
+    "_testlimitedcapi",
+    "_testmultiphase",
+    "_testsinglephase",
+];
+
 /// Convert a .tar.zst archive to an install-only .tar.gz archive.
 pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Result<W> {
     let dctx = zstd::stream::Decoder::new(reader)?;
@@ -476,6 +489,21 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
         .get("stdlib")
         .expect("stdlib entry expected");
 
+    let mut drop_paths = BTreeSet::new();
+
+    for (extension, info) in &json_main.build_info.extensions {
+        if !INSTALL_ONLY_DROP_EXTENSIONS.contains(&extension.as_str()) {
+            continue;
+        }
+
+        for entry in info {
+            if let Some(rel_path) = entry.shared_lib.as_ref() {
+                let full_path = format!("python/{}", rel_path);
+                drop_paths.insert(full_path.into_bytes());
+            }
+        }
+    }
+
     for entry in entries {
         let mut entry = entry?;
 
@@ -511,6 +539,10 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
             continue;
         }
 
+        if drop_paths.contains(&path_bytes.to_vec()) {
+            continue;
+        }
+
         let mut data = vec![];
         entry.read_to_end(&mut data)?;