Script finding unprocessable classes in other projects

.github/workflows/failures.yml

@@ -0,0 +1,46 @@
+name: Find failures
+
+permissions: {}
+
+on:
+  pull_request:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+# Only allow one instance of this workflow for each PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  failures:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        package:
+          - pretalx
+          - pretix
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41  # v7.1.2
+        with:
+          version: "latest"
+      - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3  # v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Find failure cases
+        run: just find-failures pretalx --django-settings pretalx.settings
+
+      - name: Upload findings
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: failures-${{ matrix.package }}
+          path: failures-*
+          if-no-files-found: ignore

.gitignore

@@ -3,3 +3,4 @@
 /output/
 dist
 /projects/
+failures*.md

justfile

@@ -46,15 +46,23 @@ test *args="":
 e2e *args="--console-theme dracula":
     classify tests.dummy_class.DummyClass --django-settings classify.contrib.django.settings {{ args }}
 
-find-classes path settings="":
+find-classes package settings="":
     #!/bin/bash
     set -u
 
-    class_paths=$(scripts/classes.py {{ path }} --django-settings {{ settings }})
+    class_paths=$(scripts/classes.py {{ package }} --django-settings {{ settings }})
     while read path; do
         classify --django-settings {{ settings }} "$path" > /dev/null 2>&1
         if [ $? -ne 0 ]; then
             echo $path
             continue
         fi
     done <<< "$class_paths"
+
+find-failures package *args="":
+    #!/bin/bash
+    set -euo pipefail
+
+    uv add {{ package }}
+    uv run scripts/classes.py {{ package }} {{ args }} | uv run scripts/collate_failures.py --output=failures-{{ package }}.md {{ args }}
+    uv remove {{ package }}

scripts/classes.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import ast
+import importlib
 import itertools
 import os
 from collections.abc import Iterator
@@ -30,23 +31,21 @@ def visit_ClassDef(self, node: ast.ClassDef):
         self._in_class = False
 
 
-def dotted_path(path: Path, root: Path) -> str:
-    """Convert a file path to a dotted module path."""
-    try:
-        rel_path = path.relative_to(root)
-    except ValueError:
-        return ""
+def dotted_path(path: Path, root: Path, prefix: str) -> str:
+    """Convert a file path to a dotted module path rooted at `prefix`."""
+    rel_path = path.relative_to(root)
 
     parts = list(rel_path.parts)
     if parts[-1] == "__init__.py":
         parts = parts[:-1]
     elif parts[-1].endswith(".py"):
         parts[-1] = parts[-1][:-3]
 
-    return ".".join(parts)
+    suffix = ".".join(parts)
+    return f"{prefix}.{suffix}" if suffix else prefix
 
 
-def iter_classes(path: Path, root: Path) -> Iterator[str]:
+def iter_classes(path: Path, root: Path, prefix: str) -> Iterator[str]:
     """Parse a Python file and extract all class definitions."""
     try:
         source = path.read_text(encoding="utf-8")
@@ -55,7 +54,7 @@ def iter_classes(path: Path, root: Path) -> Iterator[str]:
         click.echo(f"Warning: Could not parse {path}: {e}", err=True)
         return []
 
-    module_path = dotted_path(path, root)
+    module_path = dotted_path(path, root, prefix)
     finder = ClassFinder(str(path), module_path)
     finder.visit(tree)
     yield from finder.classes
@@ -64,22 +63,27 @@ def iter_classes(path: Path, root: Path) -> Iterator[str]:
 def iter_files(root: Path) -> Iterator[Path]:
     """Recursively find all Python files in a directory."""
     for path in root.rglob("*.py"):
-        if not any(part.startswith(".") for part in path.parts):
+        if not any(part.startswith(".") for part in path.relative_to(root).parts):
             yield path
 
 
 @click.command()
-@click.argument("path", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.argument("package")
 @click.option("--django-settings")
-def cli(path, django_settings):
+def cli(package, django_settings):
     if django_settings:
         os.environ["DJANGO_SETTINGS_MODULE"] = django_settings
         django.setup()
 
-    # find all modules
-    files = iter_files(path)
+    module = importlib.import_module(package)
+    package_path = Path(module.__path__[0])
 
-    classes = list(itertools.chain.from_iterable(iter_classes(f, path) for f in files))
+    files = iter_files(package_path)
+    classes = list(
+        itertools.chain.from_iterable(
+            iter_classes(f, package_path, package) for f in files
+        )
+    )
     for cls_path in classes:
         click.echo(cls_path)
 

scripts/collate_failures.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import logging
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+import click
+import structlog
+
+from classify.classification import classify
+from classify.django import setup_django
+from classify.resolution import resolve
+
+
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(logging.WARNING),
+)
+
+
+def write_failures(failures: dict[str, list[str]], output_path: Path) -> None:
+    sections = []
+
+    for error_type in sorted(failures):
+        classes = sorted(failures[error_type])
+        body = "\n".join(f"- {c}" for c in classes)
+        sections.append(f"## {error_type} ({len(classes)})\n{body}")
+
+    content = "\n\n".join(sections)
+    if content:
+        content += "\n"
+    output_path.write_text(content)
+
+
+@click.command()
+@click.option("--django-settings")
+@click.option(
+    "--output",
+    "output_path",
+    default="failures.md",
+    type=click.Path(path_type=Path),
+)
+def cli(django_settings, output_path):
+    if django_settings:
+        setup_django(django_settings)
+
+    failures: dict[str, list[str]] = defaultdict(set)
+
+    for line in sys.stdin:
+        class_path = line.strip()
+        if not class_path:
+            continue
+
+        try:
+            classify(resolve(class_path))
+        except Exception as exc:  # noqa: BLE001
+            error_type = type(exc).__name__
+            failures[error_type].add(class_path)
+
+    write_failures(failures, output_path)
+
+
+if __name__ == "__main__":
+    cli()