Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/fuzzing-docker-avm-build-private.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ jobs:
secrets: |
github_token=${{ secrets.GITHUB_TOKEN }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=private

- name: Install cosign
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/fuzzing-docker-avm-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ jobs:
build-args: |
COMMIT=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit || github.sha }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=public

- name: Install cosign
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/fuzzing-docker-build-private.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ jobs:
secrets: |
github_token=${{ secrets.GITHUB_TOKEN }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=private
- name: Install oras
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/fuzzing-docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,25 @@ jobs:
context: container-builds/fuzzing-container/src/
push: true
tags: |
ghcr.io/aztecprotocol/fuzzing-container:latest
ghcr.io/aztecprotocol/fuzzing-container:${{ github.sha }}
ghcr.io/aztecprotocol/contfuzzer-demo:latest
ghcr.io/aztecprotocol/contfuzzer-demo:${{ github.sha }}
build-args: |
COMMIT=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit || github.sha }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=public

- name: Install oras
uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 # v1.2.4

- name: Extract and attach fuzzer manifest
run: |
docker create --name manifest-extract ghcr.io/aztecprotocol/fuzzing-container:${{ github.sha }}
docker create --name manifest-extract ghcr.io/aztecprotocol/contfuzzer-demo:${{ github.sha }}
docker cp manifest-extract:/home/fuzzer/aztec-packages/barretenberg/cpp/fuzzer_manifest.json ./fuzzer_manifest.json
docker rm manifest-extract
oras attach ghcr.io/aztecprotocol/fuzzing-container@${{ steps.build.outputs.digest }} \
oras attach ghcr.io/aztecprotocol/contfuzzer-demo@${{ steps.build.outputs.digest }} \
--artifact-type application/vnd.aztec.fuzzer-manifest+json \
fuzzer_manifest.json

Expand All @@ -70,4 +70,4 @@ jobs:
- name: Sign image
run: |
cosign sign --yes \
ghcr.io/aztecprotocol/fuzzing-container@${{ steps.build.outputs.digest }}
ghcr.io/aztecprotocol/contfuzzer-demo@${{ steps.build.outputs.digest }}
3 changes: 2 additions & 1 deletion barretenberg/cpp/CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,8 @@
"ENABLE_ASAN": "ON",
"DISABLE_ASM": "ON",
"CMAKE_BUILD_TYPE": "RelWithAssert",
"MULTITHREADING": "OFF"
"MULTITHREADING": "OFF",
"FUZZER_PRESET_NAME": "${presetName}"
}
},
{
Expand Down
141 changes: 141 additions & 0 deletions barretenberg/cpp/scripts/flatten_and_manifest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Flatten variant binaries into /targets/ and generate a unified v2 manifest.

Reads per-preset manifests (produced by generate_fuzzer_manifest.py via CMake)
to discover binaries, suffixes, and source paths. No variant knowledge is
hardcoded here — it all comes from the manifests.

Usage (from barretenberg/cpp/ during Docker build):
python3 scripts/flatten_and_manifest.py /targets /tmp/fuzzer_manifest.json \
bin-fuzzing/fuzzer_manifest.json \
bin-fuzzing-noasm/fuzzer_manifest.json \
bin-fuzzing-avm/fuzzer_manifest.json \
--copy-only bin-fuzzing-asan/fuzzer_manifest.json \
--copy-only build-fuzzing-cov/bin/fuzzer_manifest.json

Positional manifests: binaries are copied AND added to the unified manifest.
--copy-only manifests: binaries are copied but NOT added (companion builds
like asan/coverage that the entrypoint discovers by suffix convention).
"""

import argparse
import json
import os
import shutil
import sys


def _load_manifest(path: str) -> dict:
with open(path) as f:
return json.load(f)


def _copy_executables(bin_dir: str, dest_dir: str, suffix: str) -> list[str]:
"""Copy executable files from bin_dir to dest_dir/<name><suffix>. Returns names."""
copied = []
if not os.path.isdir(bin_dir):
print(f"Warning: {bin_dir} not found, skipping", file=sys.stderr)
return copied
for name in sorted(os.listdir(bin_dir)):
src = os.path.join(bin_dir, name)
if os.path.isfile(src) and os.access(src, os.X_OK):
shutil.copy2(src, os.path.join(dest_dir, f"{name}{suffix}"))
copied.append(name)
return copied


def main():
parser = argparse.ArgumentParser(description="Flatten binaries and generate v2 manifest")
parser.add_argument("targets_dir", help="Destination for flattened binaries")
parser.add_argument("manifest_out", help="Output path for unified v2 manifest")
parser.add_argument("manifests", nargs="*", help="Per-preset manifests (schedulable)")
parser.add_argument("--copy-only", action="append", default=[], dest="companions",
help="Per-preset manifests for companion builds (copy only, not scheduled)")
parser.add_argument("--cpus", type=int, default=4, help="Default CPUs per target")
parser.add_argument("--memory-gb", type=int, default=8, help="Default memory GB per target")
args = parser.parse_args()

os.makedirs(args.targets_dir, exist_ok=True)

# Collect coverage fuzzer names (from whichever manifest has preset fuzzing-coverage)
cov_names: set[str] = set()
for path in args.manifests + args.companions:
data = _load_manifest(path)
if data.get("preset") == "fuzzing-coverage":
cov_names = {fz["name"] for fz in data.get("fuzzers", [])}

targets = []
total_copied = 0

# The fuzzing-avm preset enables FUZZING_AVM in cmake, which adds the
# AVM-specific fuzzers (avm_fuzzer_*, harness_*). However cmake also
# rebuilds every base fuzzer target (stdlib_*, ecc_*, translator_*, etc.)
# as a side effect. Those collateral rebuilds are identical in behaviour
# to the base preset and should NOT be scheduled as separate jobs.
#
# We collect fuzzer names from the first (base) manifest, then for the
# AVM preset we only schedule names that are new (the actual AVM fuzzers).
# All binaries are still copied to /targets/ — the entrypoint may need them.
#
# TODO: if cmake learns to build only AVM-specific targets in the
# fuzzing-avm preset, this filter can be removed.
base_names: set[str] = set()

for i, path in enumerate(args.manifests):
data = _load_manifest(path)
suffix = data.get("suffix", "")
preset = data.get("preset", "")
label = preset.removeprefix("fuzzing-") or "asm"
bin_dir = os.path.dirname(path)

copied = _copy_executables(bin_dir, args.targets_dir, suffix)
total_copied += len(copied)

fuzzers_meta = {fz["name"]: fz["source_path"] for fz in data.get("fuzzers", [])}

if i == 0:
base_names = set(copied)

# See docstring above — only the AVM preset needs dedup.
dedup = preset == "fuzzing-avm" and bool(base_names)

scheduled = 0
for name in copied:
if dedup and name in base_names:
continue
scheduled += 1
source_path = fuzzers_meta.get(name, "")
modes = ["fuzz", "minimize", "reproduce", "regress"]
if name in cov_names:
modes.append("coverage")
targets.append({
"name": f"{name}{suffix}",
"display_name": f"{name} ({label})",
"language": "c++",
"fuzzer_engine": "libfuzzer",
"source_path": f"barretenberg/cpp/src/barretenberg/{source_path}" if source_path else "",
"modes": modes,
"resources": {"cpus": args.cpus, "memory_gb": args.memory_gb},
})
if scheduled < len(copied):
print(f" {preset}: {scheduled} scheduled, {len(copied) - scheduled} skipped (in base preset)")

# Process companion presets: copy binaries only
for path in args.companions:
data = _load_manifest(path)
suffix = data.get("suffix", "")
bin_dir = os.path.dirname(path)
copied = _copy_executables(bin_dir, args.targets_dir, suffix)
total_copied += len(copied)

manifest = {"schema": "2", "targets": targets}
os.makedirs(os.path.dirname(args.manifest_out) or ".", exist_ok=True)
with open(args.manifest_out, "w") as f:
json.dump(manifest, f, indent=2)

print(f"Copied {total_copied} binaries to {args.targets_dir}, "
f"wrote {len(targets)} targets to {args.manifest_out}")


if __name__ == "__main__":
main()
8 changes: 7 additions & 1 deletion barretenberg/cpp/scripts/generate_fuzzer_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,15 @@ def main():
name, source_path = line.split("|", 1)
fuzzers.append({"name": name, "source_path": source_path})

# Derive binary suffix from preset name (e.g. fuzzing-noasm -> _noasm)
suffix = ""
if preset and preset != "fuzzing":
tag = preset.removeprefix("fuzzing-").replace("coverage", "cov")
suffix = f"_{tag}"

os.makedirs(os.path.dirname(output_file), exist_ok=True)
with open(output_file, "w") as f:
json.dump({"version": 1, "preset": preset, "fuzzers": fuzzers}, f, indent=2)
json.dump({"version": 1, "preset": preset, "suffix": suffix, "fuzzers": fuzzers}, f, indent=2)

print(f"Wrote {len(fuzzers)} fuzzers to {output_file} (preset: {preset})")

Expand Down
62 changes: 0 additions & 62 deletions barretenberg/cpp/scripts/merge_fuzzer_manifests.py

This file was deleted.

Loading
Loading