Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ bench: build
run-benchmarks: bench
@cmake --build "$(BUILD_DIR)" --target run_all_benchmarks -- -j$(JOBS)

bench-diff:
@python tools/scripts/compare_benchmarks.py

bench-out:
@python tools/scripts/output_benchmarks.py

tools: BUILD_TOOLS := ON
tools: build

Expand All @@ -54,9 +60,11 @@ help:
@echo " test - run ctest (RUN_TESTS=ON)"
@echo " bench - build & run benchmarks (BUILD_BENCH=ON)"
@echo " run-benchmarks - run all benchmarks"
@echo " bench-diff - compare the two most recent benchmark runs"
@echo " bench-out - output the most recent benchmark run"
@echo " tools - build project-defined tools (BUILD_TOOLS=ON)"
@echo " format - run clang-format on all source files"
@echo " clean - clean build artefacts"
@echo " clean - clean build artifacts"
@echo " distclean - clean entire build dir"
@echo ""
@echo "Knobs (override with VAR=value):"
Expand Down
58 changes: 58 additions & 0 deletions tools/scripts/compare_benchmarks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright (c) Brandon Pacewic
# SPDX-License-Identifier: MIT

import os
import re
import subprocess
from pathlib import Path
from collections import defaultdict

BUILD_DIR = "build"
RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results"
GOOGLE_BENCHMARK_COMPARE_SCRIPT = Path("../../../benchmarks/benchmark/tools/compare.py")


def extract_algo_name(filename: str) -> str:
# Removes timestamp and UUID prefix
# Matches: 2025-04-17_13-23-38_abcdef_algo.json
return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename)


def main() -> None:
print("Comparing most recent benchmarks per algorithm...")
if not RESULTS_DIR.is_dir():
print(f"Results directory '{RESULTS_DIR}' not found.")
return

algo_files = defaultdict(list)
for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True):
algo = extract_algo_name(file.name)
algo_files[algo].append(file)

for algo, files in algo_files.items():
if len(files) < 2:
print(f"Skipping {algo} (need at least 2 versions)")
continue

new, old = files[:2]
name = algo.replace(".json", "").replace(".cpp", "")

print("────────────────────────────────────────────────────────────")
print(f"Algorithm: {name}")
print("Comparing most recent two runs:")
print(f" OLD: {old.name}")
print(f" NEW: {new.name}\n")

try:
subprocess.run(
["python", str(GOOGLE_BENCHMARK_COMPARE_SCRIPT), "benchmarks", old.name, new.name],
cwd=RESULTS_DIR,
check=False,
)
except Exception as e:
print(f"Error running compare script: {e}")
print()


if __name__ == "__main__":
main()
59 changes: 59 additions & 0 deletions tools/scripts/output_benchmarks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Copyright (c) Brandon Pacewic
# SPDX-License-Identifier: MIT

import os
import re
import json
from pathlib import Path
from collections import defaultdict

BUILD_DIR = "build"
RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results"


def extract_algo_name(filename: str) -> str:
# Removes timestamp and UUID prefix
# Matches: 2025-04-17_13-23-38_abcdef_algo.json
return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename)


def print_benchmark_data(file_path: Path) -> None:
try:
with open(file_path, 'r') as f:
data = json.load(f)

for bench in data.get("benchmarks", []):
name = bench.get("name", "")
real_time = bench.get("real_time", 0.0)
cpu_time = bench.get("cpu_time", 0.0)
iterations = bench.get("iterations", 0)
print(f"{name:<30} Time: {real_time:>10.2f} CPU: {cpu_time:>10.2f} Iter: {iterations}")
except Exception as e:
print(f"Error reading {file_path.name}: {e}")


def main() -> None:
print("Showing most recent benchmark outputs per algorithm...")
if not RESULTS_DIR.is_dir():
print(f"Results directory '{RESULTS_DIR}' not found.")
return

algo_to_files = defaultdict(list)
for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True):
algo_name = extract_algo_name(file.name)
algo_to_files[algo_name].append(file)

for algo, files in algo_to_files.items():
latest = files[0]
display_name = algo.replace(".json", "").replace(".cpp", "")

print("────────────────────────────────────────────────────────────")
print(f"Algorithm: {display_name}")
print(f"File: {latest.name}\n")

print_benchmark_data(latest)
print()


if __name__ == "__main__":
main()