Skip to content

Commit bf30fa5

Browse files
ambvclaude
ambv
and
claude
committed
Add tests using production data fixtures
Real benchmark data from the production database covering a deltablue_base memory regression (~10.5% high watermark increase) between two consecutive nogil commits, while json_dumps_base and nbody_base remain unchanged. Tests verify diff regression detection, previous commit metadata, alternative metrics, trends, batch trends, filtered benchmark names, and binary/environment relationships. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7d1142e commit bf30fa5

File tree

2 files changed

+436
-0
lines changed

2 files changed

+436
-0
lines changed

‎backend/tests/production_fixtures.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
"""
2+
Test fixtures derived from the production database.
3+
4+
These represent real CPython commits, benchmark results, and the relationships
5+
between them. The deltablue_base benchmark shows a ~10.5% high watermark
6+
increase between the two commits, while json_dumps_base and nbody_base remain
7+
unchanged — a pattern typical of real-world memory regressions where only
8+
specific benchmarks are affected.
9+
"""
10+
11+
from datetime import datetime
12+
13+
BINARY_NOGIL = {
14+
"id": "nogil",
15+
"name": "Free-threaded Build",
16+
"flags": ["--disable-gil"],
17+
"description": "Experimental build without the Global Interpreter Lock (GIL).",
18+
"color": "#f59e0b",
19+
"icon": "zap",
20+
"display_order": 5,
21+
}
22+
23+
ENVIRONMENT_GH_ACTIONS = {
24+
"id": "gh_actions",
25+
"name": "GitHub actions",
26+
"description": "GitHub actions in memory.python.org",
27+
}
28+
29+
COMMIT_PREV = {
30+
"sha": "e05182f98ea100b6e26796a76b1399237aeac22f",
31+
"timestamp": datetime(2025, 8, 29, 11, 49, 35),
32+
"message": "gh-138250: load fast optimization should fall through to empty blocks (#138249)",
33+
"author": "Dino Viehland",
34+
"python_major": 3,
35+
"python_minor": 15,
36+
"python_patch": 0,
37+
}
38+
39+
COMMIT_CURR = {
40+
"sha": "d3d94e0ed715829d9bf93ef9c35e04832962f19f",
41+
"timestamp": datetime(2025, 8, 30, 22, 21, 25),
42+
"message": "gh-138061: Exclude __pycache__ directory from the computed digest in the JIT stencils (#138131)",
43+
"author": "alm",
44+
"python_major": 3,
45+
"python_minor": 15,
46+
"python_patch": 0,
47+
}
48+
49+
RUN_PREV = {
50+
"run_id": "run_e05182f9_nogil_gh_actions_1756509299",
51+
"commit_sha": COMMIT_PREV["sha"],
52+
"binary_id": "nogil",
53+
"environment_id": "gh_actions",
54+
"python_major": 3,
55+
"python_minor": 15,
56+
"python_patch": 0,
57+
"timestamp": datetime(2025, 8, 29, 23, 14, 59, 158448),
58+
}
59+
60+
RUN_CURR = {
61+
"run_id": "run_d3d94e0e_nogil_gh_actions_1756595617",
62+
"commit_sha": COMMIT_CURR["sha"],
63+
"binary_id": "nogil",
64+
"environment_id": "gh_actions",
65+
"python_major": 3,
66+
"python_minor": 15,
67+
"python_patch": 0,
68+
"timestamp": datetime(2025, 8, 30, 23, 13, 37, 215031),
69+
}
70+
71+
# deltablue_base: 10.5% high watermark increase between commits
72+
BENCH_DELTABLUE_PREV = {
73+
"id": "run_e05182f9_nogil_gh_actions_1756509299_deltablue-base",
74+
"run_id": RUN_PREV["run_id"],
75+
"benchmark_name": "deltablue_base",
76+
"high_watermark_bytes": 1_557_777,
77+
"total_allocated_bytes": 111_297_305,
78+
"allocation_histogram": [
79+
[0, 123], [3, 3612], [10, 992], [34, 61414],
80+
[111, 519085], [362, 726], [1176, 198],
81+
[3821, 386], [12416, 23], [40342, 10],
82+
],
83+
"top_allocating_functions": [
84+
{"function": "execute:deltablue_base.py:340", "count": 0, "total_size": 39_168_000},
85+
{"function": "execute:deltablue_base.py:494", "count": 0, "total_size": 23_869_728},
86+
{"function": "_get_code_from_file:<frozen runpy>:259", "count": 0, "total_size": 4_191_949},
87+
{"function": "add_propagate:deltablue_base.py:438", "count": 0, "total_size": 3_131_664},
88+
{"function": "weakest_of:deltablue_base.py:51", "count": 0, "total_size": 1_664_832},
89+
],
90+
}
91+
92+
BENCH_DELTABLUE_CURR = {
93+
"id": "run_d3d94e0e_nogil_gh_actions_1756595617_deltablue-base",
94+
"run_id": RUN_CURR["run_id"],
95+
"benchmark_name": "deltablue_base",
96+
"high_watermark_bytes": 1_721_155,
97+
"total_allocated_bytes": 111_291_390,
98+
"allocation_histogram": [
99+
[0, 123], [3, 3612], [10, 992], [34, 61399],
100+
[111, 519085], [362, 722], [1176, 197],
101+
[3821, 386], [12416, 23], [40342, 10],
102+
],
103+
"top_allocating_functions": [
104+
{"function": "execute:deltablue_base.py:340", "count": 0, "total_size": 39_168_000},
105+
{"function": "execute:deltablue_base.py:494", "count": 0, "total_size": 23_869_728},
106+
{"function": "_get_code_from_file:<frozen runpy>:259", "count": 0, "total_size": 4_191_949},
107+
{"function": "add_propagate:deltablue_base.py:438", "count": 0, "total_size": 3_131_664},
108+
{"function": "weakest_of:deltablue_base.py:51", "count": 0, "total_size": 1_664_832},
109+
],
110+
}
111+
112+
# json_dumps_base: identical across both commits
113+
BENCH_JSON_DUMPS_PREV = {
114+
"id": "run_e05182f9_nogil_gh_actions_1756509299_json-dumps-base",
115+
"run_id": RUN_PREV["run_id"],
116+
"benchmark_name": "json_dumps_base",
117+
"high_watermark_bytes": 405_465,
118+
"total_allocated_bytes": 14_132_797,
119+
"allocation_histogram": [
120+
[0, 14], [3, 425], [12, 196], [45, 49869],
121+
[160, 23501], [571, 85], [2036, 31],
122+
[7248, 22], [25805, 8], [91871, 11],
123+
],
124+
"top_allocating_functions": [
125+
{"function": "iterencode:json/encoder.py:261", "count": 0, "total_size": 7_404_609},
126+
{"function": "bench_json_dumps:json_dumps_base.py:31", "count": 0, "total_size": 1_632_536},
127+
{"function": "encode:json/encoder.py:200", "count": 0, "total_size": 1_312_456},
128+
{"function": "iterencode:json/encoder.py:252", "count": 0, "total_size": 960_240},
129+
{"function": "dumps:json/__init__.py:231", "count": 0, "total_size": 928_360},
130+
],
131+
}
132+
133+
BENCH_JSON_DUMPS_CURR = {
134+
"id": "run_d3d94e0e_nogil_gh_actions_1756595617_json-dumps-base",
135+
"run_id": RUN_CURR["run_id"],
136+
"benchmark_name": "json_dumps_base",
137+
"high_watermark_bytes": 405_465,
138+
"total_allocated_bytes": 14_132_797,
139+
"allocation_histogram": BENCH_JSON_DUMPS_PREV["allocation_histogram"],
140+
"top_allocating_functions": BENCH_JSON_DUMPS_PREV["top_allocating_functions"],
141+
}
142+
143+
# nbody_base: identical across both commits
144+
BENCH_NBODY_PREV = {
145+
"id": "run_e05182f9_nogil_gh_actions_1756509299_nbody-base",
146+
"run_id": RUN_PREV["run_id"],
147+
"benchmark_name": "nbody_base",
148+
"high_watermark_bytes": 563_371,
149+
"total_allocated_bytes": 1_808_575,
150+
"allocation_histogram": [
151+
[0, 18], [3, 1047], [10, 223], [34, 3845],
152+
[111, 804], [362, 166], [1176, 37],
153+
[3821, 75], [12416, 8], [40342, 5],
154+
],
155+
"top_allocating_functions": [
156+
{"function": "_get_code_from_file:<frozen runpy>:259", "count": 0, "total_size": 905_285},
157+
{"function": "_read_directory:<frozen zipimport>:302", "count": 0, "total_size": 132_232},
158+
{"function": "get_data:<frozen importlib._bootstrap_external>:954", "count": 0, "total_size": 132_225},
159+
{"function": "_get_code_from_file:<frozen runpy>:258", "count": 0, "total_size": 132_176},
160+
{"function": "_get_code_from_file:<frozen runpy>:254", "count": 0, "total_size": 132_176},
161+
],
162+
}
163+
164+
BENCH_NBODY_CURR = {
165+
"id": "run_d3d94e0e_nogil_gh_actions_1756595617_nbody-base",
166+
"run_id": RUN_CURR["run_id"],
167+
"benchmark_name": "nbody_base",
168+
"high_watermark_bytes": 563_371,
169+
"total_allocated_bytes": 1_808_575,
170+
"allocation_histogram": BENCH_NBODY_PREV["allocation_histogram"],
171+
"top_allocating_functions": BENCH_NBODY_PREV["top_allocating_functions"],
172+
}
173+
174+
ALL_PREV_BENCHMARKS = [BENCH_DELTABLUE_PREV, BENCH_JSON_DUMPS_PREV, BENCH_NBODY_PREV]
175+
ALL_CURR_BENCHMARKS = [BENCH_DELTABLUE_CURR, BENCH_JSON_DUMPS_CURR, BENCH_NBODY_CURR]

0 commit comments

Comments
 (0)