Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 99 additions & 12 deletions .github/scripts/fuzz_report/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import json
import os
import re
import shlex
import subprocess
import sys
import time
from pathlib import Path

from .dedup import check_duplicate
Expand Down Expand Up @@ -40,6 +42,30 @@ def _write_github_output(key: str, value: str) -> None:
f.write(f"{key}={value}\n")


def _run_gh(cmd: list[str], *, retries: int = 1, **kwargs) -> subprocess.CompletedProcess:
"""Run a gh CLI command with logging and retry on failure.

Prints the command before execution and surfaces stderr on failure.
Retries up to ``retries`` times (default 1) with a short back-off.
"""
print(f"+ {shlex.join(cmd)}", file=sys.stderr)
last_exc: subprocess.CalledProcessError | None = None
for attempt in range(1 + retries):
if attempt > 0:
wait = 5 * attempt
print(f"Retrying in {wait}s (attempt {attempt + 1}/{1 + retries})...", file=sys.stderr)
time.sleep(wait)
try:
return subprocess.run(cmd, check=True, **kwargs)
except subprocess.CalledProcessError as exc:
last_exc = exc
stderr_text = (
exc.stderr if isinstance(exc.stderr, str) else (exc.stderr or b"").decode()
)
print(f"gh command failed (exit {exc.returncode}): {stderr_text}", file=sys.stderr)
raise last_exc # type: ignore[misc]


def _load_crash_info(path: str | Path) -> CrashInfo:
"""Load CrashInfo from a JSON file."""
crash_data = json.loads(Path(path).read_text())
Expand All @@ -53,6 +79,21 @@ def _find_crash_file(crash_dir: str, crash_name: str) -> str | None:
return None


def _truncate(text: str, max_chars: int) -> str:
"""Truncate text to max_chars, appending a note if truncated."""
if len(text) <= max_chars:
return text
return text[:max_chars] + f"\n... ({len(text) - max_chars} chars truncated)"


# GitHub issue body limit is 65536 chars. Reserve ~5k for the fixed template
# chrome (headings, summary table, reproduction steps, etc.) and split the
# remaining budget between the two variable-length fields.
_BODY_BUDGET = 60000
_TEMPLATE_OVERHEAD = 5000
_FIELD_BUDGET = _BODY_BUDGET - _TEMPLATE_OVERHEAD # 55k split between the two


def _build_template_variables(
crash_info: CrashInfo,
var_args: list[tuple[str, str]] | None = None,
Expand All @@ -64,11 +105,28 @@ def _build_template_variables(
for key, value in var_args:
variables[key] = value

panic_msg = crash_info.panic_message
stack_trace = crash_info.stack_trace_raw

# Truncate the two large fields so their combined size fits the budget.
combined = len(panic_msg) + len(stack_trace)
if combined > _FIELD_BUDGET:
# Give panic_message up to half, stack_trace gets the rest.
msg_limit = min(len(panic_msg), _FIELD_BUDGET // 2)
trace_limit = _FIELD_BUDGET - msg_limit
panic_msg = _truncate(panic_msg, msg_limit)
stack_trace = _truncate(stack_trace, trace_limit)
print(
f"Warning: Truncated issue fields to fit body limit "
f"(panic_message={msg_limit}, stack_trace={trace_limit})",
file=sys.stderr,
)

# Auto-populate from crash info (don't override explicit -v args)
auto_vars = {
"PANIC_MESSAGE": crash_info.panic_message,
"PANIC_MESSAGE": panic_msg,
"CRASH_LOCATION": crash_info.crash_location,
"STACK_TRACE_RAW": crash_info.stack_trace_raw,
"STACK_TRACE_RAW": stack_trace,
"DEBUG_OUTPUT": crash_info.debug_output,
"SEED_HASH": crash_info.seed_hash,
"STACK_TRACE_HASH": crash_info.stack_trace_hash,
Expand Down Expand Up @@ -128,7 +186,7 @@ def _update_recurrence_count(repo: str, issue_number: int | str) -> int:
Returns the new count.
"""
# List all comments on the issue
result = subprocess.run(
result = _run_gh(
[
"gh",
"api",
Expand All @@ -139,7 +197,6 @@ def _update_recurrence_count(repo: str, issue_number: int | str) -> int:
],
capture_output=True,
text=True,
check=True,
)

existing_id = None
Expand All @@ -161,7 +218,7 @@ def _update_recurrence_count(repo: str, issue_number: int | str) -> int:
if existing_id:
# Update existing comment (not atomic — race is acceptable since
# fuzz CI jobs are serialized)
subprocess.run(
_run_gh(
[
"gh",
"api",
Expand All @@ -171,19 +228,17 @@ def _update_recurrence_count(repo: str, issue_number: int | str) -> int:
"-f",
f"body={body}",
],
check=True,
)
else:
# Create new recurrence comment
subprocess.run(
_run_gh(
[
"gh",
"api",
f"repos/{repo}/issues/{issue_number}/comments",
"-f",
f"body={body}",
],
check=True,
)

return new_count
Expand Down Expand Up @@ -302,7 +357,7 @@ def cmd_report(args: argparse.Namespace) -> int:
body_file = Path("comment_body.md")
body_file.write_text(body)

subprocess.run(
_run_gh(
[
"gh",
"issue",
Expand All @@ -313,7 +368,6 @@ def cmd_report(args: argparse.Namespace) -> int:
"--body-file",
str(body_file),
],
check=True,
)
print(f"Commented on #{existing_issue}", file=sys.stderr)
_write_github_output("issue_number", str(existing_issue))
Expand All @@ -325,7 +379,11 @@ def cmd_report(args: argparse.Namespace) -> int:
body_file = Path("issue_body.md")
body_file.write_text(body)

result = subprocess.run(
print(f"Issue title: {title}", file=sys.stderr)
print(f"Issue body size: {len(body)} chars", file=sys.stderr)
print(f"Repo: {args.repo}", file=sys.stderr)

result = _run_gh(
[
"gh",
"issue",
Expand All @@ -339,7 +397,6 @@ def cmd_report(args: argparse.Namespace) -> int:
"--body-file",
str(body_file),
],
check=True,
capture_output=True,
text=True,
)
Expand All @@ -349,6 +406,36 @@ def cmd_report(args: argparse.Namespace) -> int:
print(f"Created issue #{issue_number}: {issue_url}", file=sys.stderr)
_write_github_output("issue_number", issue_number)

# Post full debug output as a follow-up comment (collapsed).
debug_output = variables.get("DEBUG_OUTPUT", "")
if debug_output and debug_output != "(not set)":
comment_body = (
"<details>\n<summary>Debug Output</summary>\n\n"
f"```\n{debug_output}\n```\n</details>"
)
# Truncate comment to GitHub's limit too.
if len(comment_body) > _BODY_BUDGET:
comment_body = (
comment_body[: _BODY_BUDGET - 50] + "\n```\n</details>\n\n*Truncated*"
)
comment_file = Path("debug_comment.md")
comment_file.write_text(comment_body)
try:
_run_gh(
[
"gh",
"issue",
"comment",
issue_number,
"--repo",
args.repo,
"--body-file",
str(comment_file),
],
)
except subprocess.CalledProcessError:
print("Warning: failed to post debug output comment", file=sys.stderr)

return 0


Expand Down
20 changes: 0 additions & 20 deletions .github/scripts/fuzz_report/dedup.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,26 +176,6 @@ def check_error_pattern(message_hash: str, error_variant: str, issues: list[dict
},
)

# Second try: same error variant (lower confidence)
if error_variant and error_variant != "unknown":
for issue in issues:
body = issue.get("body", "")
if error_variant in body:
return DedupResult(
duplicate=True,
check="error_pattern",
confidence="medium",
issue_number=issue["number"],
issue_url=issue.get("url"),
issue_title=issue.get("title"),
reason=f"Same error variant type: {error_variant}",
debug={
"message_hash": message_hash,
"error_variant": error_variant,
"matched_issue": issue["number"],
},
)

return DedupResult(
duplicate=False,
check="error_pattern",
Expand Down
15 changes: 4 additions & 11 deletions .github/scripts/fuzz_report/templates/new_issue.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,19 @@
{{PANIC_MESSAGE}}
```

**Stack Trace**:
<details>
<summary>Stack Trace</summary>

```
{{STACK_TRACE_RAW}}
```
</details>
{% if CLAUDE_ANALYSIS %}

### Root Cause Analysis

{{CLAUDE_ANALYSIS}}
{% endif %}
{% if DEBUG_OUTPUT %}

<details>
<summary>Debug Output</summary>

```
{{DEBUG_OUTPUT}}
```
</details>
{% endif %}

### Summary

Expand Down
6 changes: 2 additions & 4 deletions .github/scripts/fuzz_report/tests/test_dedup.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,15 +144,13 @@ def test_message_hash_match(self):
assert result.duplicate is True
assert result.confidence == "high"

def test_variant_match(self):
def test_variant_only_does_not_match(self):
result = check_error_pattern(
"nomatchhash",
"ScalarMismatch",
EXISTING_ISSUES,
)
assert result.duplicate is True
assert result.confidence == "medium"
assert result.issue_number == 101
assert result.duplicate is False

def test_no_match(self):
result = check_error_pattern("nomatch", "UnknownVariant", EXISTING_ISSUES)
Expand Down
8 changes: 4 additions & 4 deletions .github/scripts/fuzz_report/tests/test_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,12 @@ def test_claude_analysis_hidden_when_empty(self, new_issue_template):
rendered = render_template(new_issue_template, vars_no_analysis, use_env=False)
assert "Root Cause Analysis" not in rendered

def test_debug_output_in_details(self, new_issue_template):
"""Debug output should be inside a <details> block."""
def test_stack_trace_in_details(self, new_issue_template):
"""Stack trace should be inside a <details> block."""
rendered = render_template(new_issue_template, self.SAMPLE_VARS, use_env=False)
assert "<details>" in rendered
assert "Debug Output" in rendered
assert "Array { dtype: Int32, len: 10 }" in rendered
assert "Stack Trace" in rendered
assert "Debug Output" not in rendered

def test_related_comment_target_pattern(self, related_comment_template):
"""Related comment template should also have compatible Target pattern."""
Expand Down
21 changes: 20 additions & 1 deletion .github/workflows/report-fuzz-crash.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,32 @@ jobs:
run: pip install -e .github/scripts

- name: Extract crash info
id: extract
run: |
python3 -m fuzz_report extract \
logs/fuzz_output.log \
--crash-dir crash_artifacts \
--crash-name "${{ inputs.crash_file }}" \
-o crash_info.json

# Validate that extraction found a real crash.
error_variant=$(jq -r '.error_variant' crash_info.json)
panic_message=$(jq -r '.panic_message' crash_info.json)
echo "error_variant=$error_variant"
echo "panic_message=$panic_message"
if [ "$error_variant" = "unknown" ] && [ "$panic_message" = "unknown" ]; then
echo "::notice::No crash info found in fuzzer output — nothing to report."
echo "crash_found=false" >> "$GITHUB_OUTPUT"
else
echo "crash_found=true" >> "$GITHUB_OUTPUT"
fi

- name: Skip if no crash found
if: steps.extract.outputs.crash_found != 'true'
run: |
echo "No parseable crash in fuzzer output — skipping remaining steps."
exit 1

- name: Fetch existing fuzzer issues
env:
GH_TOKEN: ${{ secrets.gh_token }}
Expand Down Expand Up @@ -116,7 +135,7 @@ jobs:
claude_args: |
--model claude-opus-4-6
--max-turns 5
--allowedTools "Read,Write,Bash(cat:*),Bash(jq:*)"
--allowedTools "Read,Write,Grep,Glob,Bash(cat:*),Bash(jq:*),Bash(head:*),Bash(xxd:*)"

- name: Create or comment on issue
id: report
Expand Down
Loading