Refine crossref check and improve test cases

Author: mathpluscode

.claude-plugin/marketplace.json

@@ -16,7 +16,7 @@
         "path": "./"
       },
       "description": "A bibliography toolkit for LaTeX",
-      "version": "1.5.1",
+      "version": "1.6.0",
       "keywords": ["bibtex", "bibliography", "latex", "overleaf", "academic", "reference", "citation"],
       "category": "academic",
       "license": "MIT"

.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "bibtools",
   "description": "A bibliography toolkit for LaTeX",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "author": {
     "name": "Yunguan Fu"
   },

docs/build.py

@@ -409,7 +409,7 @@ def build_html(cards_html: str) -> str:
     font-size: 0.75rem;
   }}
 
-  .diff-line {{ padding: 0 1rem; white-space: pre; }}
+  .diff-line {{ padding: 0 1rem; white-space: pre; display: block; min-width: fit-content; }}
   .diff-line.del {{ background: var(--del-bg); color: var(--del-line); }}
   .diff-line.add {{ background: var(--add-bg); color: var(--add-line); }}
   .diff-line.ctx {{ color: var(--text-muted); }}

docs/index.html

@@ -222,7 +222,7 @@
     font-size: 0.75rem;
   }
 
-  .diff-line { padding: 0 1rem; white-space: pre; }
+  .diff-line { padding: 0 1rem; white-space: pre; display: block; min-width: fit-content; }
   .diff-line.del { background: var(--del-bg); color: var(--del-line); }
   .diff-line.add { background: var(--add-bg); color: var(--add-line); }
   .diff-line.ctx { color: var(--text-muted); }
@@ -402,7 +402,7 @@ <h2 class="section-title">Examples</h2>
 <div class="diff-card">
   <div class="diff-header">
     <span class="diff-title">Duplicate pair (bioRxiv preprint + published version)</span>
-    <span class="stats"><span class="add-count">+1</span></span>
+    <span class="stats"><span class="add-count">+4</span> <span class="del-count">-2</span></span>
     <span class="diff-badge badge-duplicate">duplicate detected</span>
   </div>
   <div class="diff-body">
@@ -418,12 +418,15 @@ <h2 class="section-title">Examples</h2>
     <div class="diff-line ctx"> </div>
     <div class="diff-line ctx"> @article{watson2023novo,</div>
     <div class="diff-line ctx">   title={De novo design of protein structure and function with RFdiffusion},</div>
-    <div class="diff-line ctx">   author={Watson, Joseph L and Juergens, David and Bennett, Nathaniel R and Trippe, Brian L and Yim, Jason and Eisenach, Helen E and Ahern, Woody and Borst, Andrew J and Ragotte, Robert J and Milles, Lukas F and others},</div>
+    <div class="diff-line del">-  author={Watson, Joseph L and Juergens, David and Bennett, Nathaniel R and Trippe, Brian L and Yim, Jason and Eisenach, Helen E and Ahern, Woody and Borst, Andrew J and Ragotte, Robert J and Milles, Lukas F and others},</div>
+    <div class="diff-line add">+  author={Watson, Joseph L. and Juergens, David and Bennett, Nathaniel R. and Trippe, Brian L. and Yim, Jason and Eisenach, Helen E. and Ahern, Woody and Borst, Andrew J. and Ragotte, Robert J. and Milles, Lukas F. and Wicky, Basile I. M. and Hanikel, Nikita and Pellock, Samuel J. and Courbet, Alexis and Sheffler, William and Wang, Jue and Venkatesh, Preetham and Sappington, Isaac and Torres, Susana Vázquez and Lauko, Anna and De Bortoli, Valentin and Mathieu, Emile and Ovchinnikov, Sergey and Barzilay, Regina and Jaakkola, Tommi S. and DiMaio, Frank and Baek, Minkyung and Baker, David},</div>
     <div class="diff-line ctx">   journal={Nature},</div>
     <div class="diff-line ctx">   volume={620},</div>
     <div class="diff-line ctx">   pages={1089--1100},</div>
     <div class="diff-line ctx">   year={2023},</div>
-    <div class="diff-line ctx">   publisher={Nature Publishing Group UK London}</div>
+    <div class="diff-line del">-  publisher={Nature Publishing Group UK London}</div>
+    <div class="diff-line add">+  publisher={Nature Publishing Group UK London},</div>
+    <div class="diff-line add">+  number={7976}</div>
     <div class="diff-line ctx"> }</div>
   </div>
 </div>

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bibtools"
-version = "1.5.1"
+version = "1.6.0"
 description = "A bibliography toolkit for LaTeX, built as agent skills"
 requires-python = ">=3.10"
 license = "MIT"

skills/bibtidy/SKILL.md

@@ -27,6 +27,7 @@ Assume standard brace-style BibTeX entries like `@article{...}`. Parenthesized B
 | **Field comparison** | `python3 $TOOLS_DIR/compare.py <file.bib> [--key KEY]` |
 | CrossRef DOI lookup | `python3 $TOOLS_DIR/crossref.py doi <DOI>` |
 | CrossRef title search | `python3 $TOOLS_DIR/crossref.py search "<title>"` |
+| CrossRef bibliographic search | `python3 $TOOLS_DIR/crossref.py bibliographic "<query>"` |
 | Duplicate detection | `python3 $TOOLS_DIR/duplicates.py <file.bib>` |
 | **Apply edits** | `python3 $TOOLS_DIR/edit.py <file.bib> <patches.json>` |
 | Web verification | web search (preferred) or CrossRef scripts (fallback) |
@@ -84,10 +85,10 @@ For unchanged entries, do NOT add any comments or URLs.
 2. Back up for format validation: `cp <file>.bib <file>.bib.orig`
 3. Preserve `@string`, `@preamble`, `@comment` blocks verbatim
 4. Run duplicate detection: `python3 $TOOLS_DIR/duplicates.py <file.bib>`
-5. **Run field comparison**: `python3 $TOOLS_DIR/compare.py <file.bib>` — this programmatically compares every entry against CrossRef and returns exact field-level mismatches. Do NOT skip this step or rely on visual comparison alone. The output is a JSON list; each element has `key`, `versions` (a list of CrossRef matches, each with `mismatches`, `url`, `doi`, etc.), and `error`. **Skip rule**: if an entry has zero mismatches across all versions and no error in the compare.py output, skip it entirely — do NOT investigate, modify, or add comments to it. Only proceed with entries that compare.py flagged (mismatches, errors, or duplicates from step 4).
-6. **Verify every planned modification with web search** — for entries that compare.py flagged with mismatches or errors, and for entries flagged as duplicates, verify the planned action via web search. For `fix` patches, gather one or more source URLs. Entries where `compare.py` returned an error (e.g. "No exact title match") still need full verification — the verification agent should search for the paper and check all fields. **Important: verification agents MUST NOT override `compare.py` field values.** CrossRef is the authoritative source for metadata (pages, volume, number, etc.) because it receives data directly from publishers via DOI registration. When web search finds a conflicting value (e.g. different page numbers on a conference website), always use the CrossRef value and add `% bibtidy: REVIEW` if desired — but do NOT keep the old value.
+5. **Run field comparison**: `python3 $TOOLS_DIR/compare.py <file.bib>` — this programmatically compares every entry against CrossRef and returns exact field-level mismatches. Do NOT skip this step or rely on visual comparison alone. The output is a JSON list; each element has `key`, `versions` (a list of alternative CrossRef candidate matches for the same entry, each with `mismatches`, `url`, `doi`, etc.), and `error`. When multiple versions are returned, choose the best matching candidate; do not combine fields from different versions. **Skip rule**: if an entry has zero mismatches across all versions and no error in the compare.py output, skip it entirely — do NOT investigate, modify, or add comments to it. Only proceed with entries that compare.py flagged (mismatches, errors, or duplicates from step 4).
+6. **Verify every planned modification with web search** — for entries that compare.py flagged with mismatches or errors, and for entries flagged as duplicates, verify the planned action via web search. For `fix` patches, gather one or more source URLs. Entries where `compare.py` returned an error (e.g. "No exact title match") still need full verification — the verification agent should search for the paper and check all fields. **Important: after selecting the best-matching version, verification agents MUST NOT override that selected version's `compare.py` field values.** CrossRef is the authoritative source for metadata (pages, volume, number, etc.) because it receives data directly from publishers via DOI registration. When web search finds a conflicting value (e.g. different page numbers on a conference website), always use the CrossRef value and add `% bibtidy: REVIEW` if desired — but do NOT keep the old value.
 7. **Flag hallucinated/non-existent references** — if compare.py returned an error (e.g. "No CrossRef results found" or "No exact title match in CrossRef results") AND web search also finds no matching paper, the reference likely does not exist. Add `% bibtidy: NOT FOUND — no matching paper on CrossRef or web search; verify this reference exists` above the entry, then comment out the entire entry (prefix every line with `% `). Do NOT add a URL line.
-8. Apply fixes **sequentially** using `edit.py` — do NOT edit the .bib file directly with agent editing tools (for example, Claude Code Edit or Codex `apply_patch`), and do NOT rewrite the entire file. Build a patches.json for each entry (or batch) and run `python3 $TOOLS_DIR/edit.py <file.bib> <patches.json>`. This ensures the commented original, source URLs, and explanation are always included. You MUST apply **every** mismatch reported by `compare.py` — do not skip any field (including `number`, `pages`, `volume`). Use the `crossref_value` exactly as given (do NOT rephrase, reformat, or partially apply it). For title mismatches on preprint→published upgrades, replace the entire title with the CrossRef title — do NOT try to edit parts of the old title. Never reject a CrossRef value because another source disagrees. Every patch MUST include `urls` (list of source URLs) and `explanation` (what changed and why). Include the CrossRef URL from compare.py's `url` field when available, plus any other authoritative source (DOI URL, venue page) found via web search.
+8. Apply fixes **sequentially** using `edit.py` — do NOT edit the .bib file directly with agent editing tools (for example, Claude Code Edit or Codex `apply_patch`), and do NOT rewrite the entire file. Build a patches.json for each entry (or batch) and run `python3 $TOOLS_DIR/edit.py <file.bib> <patches.json>`. This ensures the commented original, source URLs, and explanation are always included. After selecting the correct version, you MUST apply **every** mismatch from that selected version — do not skip any field (including `number`, `pages`, `volume`). Use the `crossref_value` exactly as given (do NOT rephrase, reformat, or partially apply it). For title mismatches on preprint→published upgrades, replace the entire title with the CrossRef title — do NOT try to edit parts of the old title. Never reject a CrossRef value because another source disagrees. Every patch MUST include `urls` (list of source URLs) and `explanation` (what changed and why). Include the CrossRef URL from compare.py's `url` field when available, plus any other authoritative source (DOI URL, venue page) found via web search.
 9. Run format validation; fix violations and re-run until clean
 10. Delete backup: `rm <file>.bib.orig`
 11. Print a Markdown summary table with headers `Metric | Count` and exactly these rows: total entries, verified, fixed, not found. Do NOT include a separate "needs manual review" row.
@@ -103,7 +104,7 @@ Use subagents, when available, to verify multiple entries concurrently. This dra
 
 **When CrossRef fails**, find the paper's official venue page via web search. Many venues (JMLR, NeurIPS, CVPR, etc.) provide a downloadable `.bib` file — fetch it directly when possible. An official `.bib` is the most reliable source: it has exact title, authors, volume, number, and pages with no guessing.
 
-Launch verification subagents in one batch so they run concurrently. Group into batches of ~10 if there are many entries.
+Launch verification subagents in one batch so they run concurrently. Cap at **6 subagents** and distribute entries evenly across them (e.g., 18 entries = 3 per subagent, 60 entries = 10 per subagent). For ≤6 entries, use one subagent per entry. If the user explicitly requests more parallelism, you may increase beyond 6.
 
 **Step 2 — Collect results:** Read each agent's returned summary.
 
@@ -145,7 +146,7 @@ For each `@article`, `@inproceedings`, `@book`, etc.:
 
 **1. Verify existence** — Search for `"<title>" <first author last name>`. If not found: `% bibtidy: NOT FOUND — verify manually`
 
-**2. Cross-check metadata** — If DOI exists, fetch via `crossref.py doi <DOI>`. Otherwise `crossref.py search "<title>"`. Compare title, year, authors, journal, volume, pages.
+**2. Cross-check metadata** — Always search via `crossref.py search "<title>"`. If DOI exists, also fetch via `crossref.py doi <DOI>`. If neither finds a match, fall back to `crossref.py bibliographic "<title>"`. Compare title, year, authors, journal, volume, number, pages.
 
 **3. Check for published preprints** — If journal contains "arxiv"/"biorxiv"/"chemrxiv", search for published version. Update title, venue, year, volume, pages, entry type. Only update if confirmed via DOI or two independent sources.
 

skills/bibtidy/tools/compare.py

@@ -17,7 +17,7 @@
 import re
 import sys
 
-from crossref import fetch_doi, search_title
+from crossref import fetch_doi, search_bibliographic, search_title
 from duplicates import normalize_doi, normalize_title, parse_bib_entries, split_bibtex_authors
 
 
@@ -179,34 +179,44 @@ def lookup_and_compare(entry: dict, timeout: int = 10) -> dict:
     key = entry["key"]
     result = {"key": key, "versions": [], "error": None}
 
-    # Try DOI first, then title search
+    title = entry.get("title", "")
     doi = entry.get("doi", "").strip()
-    if doi:
-        doi = normalize_doi(doi)
-        cr = fetch_doi(doi, timeout=timeout)
-    else:
-        title = entry.get("title", "")
-        if not title:
-            result["error"] = "No DOI or title to search"
-            return result
-        cr = search_title(title, rows=3, timeout=timeout)
-
-    if "error" in cr:
-        result["error"] = cr["error"]
+    if not title and not doi:
+        result["error"] = "No DOI or title to search"
         return result
 
-    if "results" in cr:
-        items = cr["results"]
-        if not items:
-            result["error"] = "No CrossRef results found"
-            return result
-        bib_title_norm = normalize_title(entry.get("title", ""))
-        matches = [item for item in items if normalize_title(item.get("title") or "") == bib_title_norm]
-        if not matches:
-            result["error"] = "No exact title match in CrossRef results"
-            return result
-    else:
-        matches = [cr]
+    # Collect CrossRef results from multiple strategies.
+    matches = []
+    last_error = None
+    bib_title_norm = normalize_title(title)
+
+    def _search_and_filter(search_fn, query):
+        nonlocal last_error
+        cr = search_fn(query, rows=3, timeout=timeout)
+        if "error" in cr:
+            last_error = cr["error"]
+            return []
+        return [item for item in cr.get("results", [])
+                if normalize_title(item.get("title") or "") == bib_title_norm]
+
+    if title:
+        matches = _search_and_filter(search_title, title)
+
+    if doi:
+        cr = fetch_doi(normalize_doi(doi), timeout=timeout)
+        if "error" in cr:
+            last_error = cr["error"]
+        else:
+            existing_dois = {m.get("doi") for m in matches}
+            if cr.get("doi") not in existing_dois:
+                matches.append(cr)
+
+    if not matches and title:
+        matches = _search_and_filter(search_bibliographic, title)
+
+    if not matches:
+        result["error"] = last_error or "No exact title match in CrossRef results"
+        return result
 
     result["versions"] = [
         {

skills/bibtidy/tools/crossref.py

@@ -2,8 +2,9 @@
 """CrossRef API utilities for BibTeX validation.
 
 Usage:
-    python3 crossref.py doi <DOI>            — fetch metadata for a specific DOI
-    python3 crossref.py search "<title>"     — search by title, return top 3 results
+    python3 crossref.py doi <DOI>                  — fetch metadata for a specific DOI
+    python3 crossref.py search "<title>"           — search by title, return top 3 results
+    python3 crossref.py bibliographic "<query>"    — broad bibliographic search, return top 3 results
 
 Options:
     --timeout SECONDS   HTTP timeout (default: 10)
@@ -132,9 +133,9 @@ def fetch_doi(doi: str, timeout: int = 10) -> dict:
         return {"error": f"Malformed response from CrossRef: {exc}"}
 
 
-def search_title(title: str, rows: int = 3, timeout: int = 10) -> dict:
-    """Search CrossRef by title, returning up to `rows` results."""
-    params = urllib.parse.urlencode({"query.bibliographic": title, "rows": rows, "mailto": MAILTO})
+def _search(param_name: str, query: str, rows: int, timeout: int) -> dict:
+    """Search CrossRef works by a given query parameter."""
+    params = urllib.parse.urlencode({param_name: query, "rows": rows, "mailto": MAILTO})
     url = f"{CROSSREF_API}/works?{params}"
     result = _safe_fetch(url, timeout)
     if "error" in result:
@@ -146,6 +147,16 @@ def search_title(title: str, rows: int = 3, timeout: int = 10) -> dict:
         return {"error": f"Malformed response from CrossRef: {exc}"}
 
 
+def search_title(title: str, rows: int = 3, timeout: int = 10) -> dict:
+    """Search CrossRef by title, returning up to `rows` results."""
+    return _search("query.title", title, rows, timeout)
+
+
+def search_bibliographic(query: str, rows: int = 3, timeout: int = 10) -> dict:
+    """Search CrossRef by general bibliographic query, returning up to `rows` results."""
+    return _search("query.bibliographic", query, rows, timeout)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="CrossRef API utilities for BibTeX validation")
     parser.add_argument("--timeout", type=int, default=10, help="HTTP request timeout in seconds")
@@ -157,12 +168,17 @@ def main() -> None:
     search_parser = subparsers.add_parser("search", help="Search by title")
     search_parser.add_argument("title", help="Title string to search for")
 
+    bib_parser = subparsers.add_parser("bibliographic", help="Search by bibliographic query")
+    bib_parser.add_argument("query", help="Bibliographic query string")
+
     args = parser.parse_args()
 
     if args.command == "doi":
         result = fetch_doi(args.doi_value, timeout=args.timeout)
     elif args.command == "search":
         result = search_title(args.title, timeout=args.timeout)
+    elif args.command == "bibliographic":
+        result = search_bibliographic(args.query, timeout=args.timeout)
     else:
         parser.print_help()
         sys.exit(1)