#!/usr/bin/env python3 """Build chapter packets for semantic citation-reference validation. The existing bibliography checks prove that citekeys resolve. These packets support the harder editorial question: whether the cited source actually backs the sentence or paragraph where it is used. """ from __future__ import annotations import argparse import json import re from collections import defaultdict from pathlib import Path from typing import Any def find_repo_root() -> Path: """Return the nearest ancestor of this file that contains ``books``; raise ``RuntimeError`` if none does.""" for parent in Path(__file__).resolve().parents: if (parent / "books").exists(): return parent raise RuntimeError("Could not locate repository root containing books") ROOT = find_repo_root() SCOPES = [ { "name": "vol1", "paths": ["books/vol1/"], "bibs": ["books/references.bib"], }, { "name": "vol2", "paths": ["books/vol2/"], "bibs": ["books/references.bib"], }, { "name": "book-shared", "paths": [ "books/shared/frontmatter/", "books/shared/backmatter/", ], "bibs": ["books/references.bib"], }, { "name": "staffml", "paths": ["staffml/"], "bibs": ["staffml/paper/references.bib"], }, { "name": "tinytorch", "paths": ["tinytorch/"], "bibs": ["tinytorch/paper/references.bib"], }, { "name": "mlsysim", "paths": ["mlsysim/"], "bibs": ["mlsysim/paper/references.bib", "mlsysim/docs/references.bib"], }, ] QMD_ROOTS = [ "books", "staffml", "tinytorch", "mlsysim", ] EXCLUDE_PARTS = { "_build", "_site", "node_modules", ".git", "__pycache__", ".venv", "venv", ".aiconfigs-local", } NON_CITE_PREFIXES = ( "sec-", "fig-", "tbl-", "eq-", "lst-", "exr-", "exm-", "thm-", "cor-", "cnj-", "def-", "prp-", "rem-", "prf-", "alg-", "algo-", ) KNOWN_FALSE_POSITIVE_KEYS = { "media", "keyframes", "import", "supports", "page", "font-face", "charset", "namespace", "document", "grad", "staticmethod", "classmethod", "property", "abstractmethod", "dataclass", "cached_property", "wraps", "eecs", } CITE_RE = re.compile(r"(?\w+)\s*\{\s*(?P[\w:.-]+)\s*,", re.M) FENCE_RE = re.compile(r"^\s*(```|~~~)") SINGLE_LETTER_RE = re.compile(r"^[A-Z](\.[A-Z])*$") FIELDS_TO_SUMMARIZE = ( "author", "title", "booktitle", "journal", "publisher", "institution", "year", "pages", "doi", "url", ) def rel(path: Path) -> str: """Return ``path`` as a POSIX path relative to the repo root; raise ``ValueError`` outside it.""" return path.relative_to(ROOT).as_posix() def display_path(path: Path) -> str: """Return ``path`` relative to the repo root when possible, otherwise its POSIX form unchanged.""" try: return path.relative_to(ROOT).as_posix() except ValueError: return path.as_posix() def find_scope(rel_path: str) -> dict[str, Any] | None: """Return the first ``SCOPES`` entry with a path prefix matching ``rel_path``, or None.""" for scope in SCOPES: if any(rel_path.startswith(prefix) for prefix in scope["paths"]): return scope return None def discover_qmds(all_qmd: bool) -> list[Path]: """Return the sorted ``.qmd`` files to scan. Build, vendor, and cache directories (``EXCLUDE_PARTS``) and ``_shelved*`` files are skipped. With ``all_qmd`` the whole repo is walked; otherwise only files under ``QMD_ROOTS`` that fall inside a ``SCOPES`` entry count. """ roots = [ROOT] if all_qmd else [ROOT / root for root in QMD_ROOTS] files: list[Path] = [] for root in roots: if not root.exists(): continue for path in root.rglob("*.qmd"): if any(part in EXCLUDE_PARTS for part in path.parts): continue if path.name.startswith("_shelved"): continue if not all_qmd and find_scope(rel(path)) is None: continue files.append(path) return sorted(set(files)) def strip_inline_protected(line: str) -> str: """Remove inline code spans and single-line HTML comments so their ``@`` tokens are not read as citations.""" line = re.sub(r"`[^`]*`", "", line) line = re.sub(r"", "", line) return line def should_skip_key(key: str) -> bool: """Return True for ``@`` tokens that are not citekeys. Skips empty keys, cross-reference labels (``NON_CITE_PREFIXES``), known CSS at-rule and Python decorator words, capital initials, and version-like numbers. """ key = key.rstrip(".,;:)") lower_key = key.lower() return ( not key or lower_key.startswith(NON_CITE_PREFIXES) or lower_key in KNOWN_FALSE_POSITIVE_KEYS or bool(SINGLE_LETTER_RE.match(key)) or bool(re.match(r"^\d+\.\d+", key)) ) def paragraph_at(lines: list[str], line_no: int, max_chars: int) -> str: """Return the blank-line-delimited paragraph around 1-based ``line_no``, joined onto one line and cut to ``max_chars``.""" idx = max(0, min(line_no - 1, len(lines) - 1)) start = idx while start > 0 and lines[start - 1].strip(): start -= 1 end = idx while end + 1 < len(lines) and lines[end + 1].strip(): end += 1 text = " ".join(line.strip() for line in lines[start : end + 1]).strip() return text[:max_chars] def citation_occurrences(path: Path, max_context_chars: int) -> list[dict[str, Any]]: """Return one record per ``@key`` citation found in a QMD file. YAML front matter, HTML comments, ``" not in lower: in_raw_block = "style" continue if "" not in lower: in_raw_block = "script" continue if FENCE_RE.match(line): in_fence = not in_fence continue if in_fence: continue scan_line = strip_inline_protected(line) for match in CITE_RE.finditer(scan_line): key = match.group(1).rstrip(".,;:)") if should_skip_key(key): continue occurrences.append( { "occurrence_id": f"{rel(path)}:{line_no}:{match.start()}:{key}", "key": key, "line": line_no, "column": match.start() + 1, "line_text": line.strip(), "context": paragraph_at(lines, line_no, max_context_chars), } ) return occurrences def extract_bib_value(entry_text: str, field: str) -> str | None: """Return one field's value from raw BibTeX entry text, or None. Handles brace-delimited (including nested) and double-quoted values, collapses whitespace, and strips all braces. Bare values such as ``year = 2020`` are not matched and return None. """ match = re.search(rf"(?im)^\s*{re.escape(field)}\s*=\s*([{{\"])", entry_text) if not match: return None opener = match.group(1) start = match.end() if opener == '"': end = start escaped = False while end < len(entry_text): char = entry_text[end] if char == '"' and not escaped: break escaped = char == "\\" and not escaped if char != "\\": escaped = False end += 1 raw = entry_text[start:end] else: depth = 1 end = start while end < len(entry_text) and depth: char = entry_text[end] if char == "{": depth += 1 elif char == "}": depth -= 1 end += 1 raw = entry_text[start : max(start, end - 1)] raw = re.sub(r"\s+", " ", raw).strip() return raw.replace("{", "").replace("}", "") or None def parse_bib_entries(bib_paths: list[str]) -> dict[str, list[dict[str, Any]]]: """Index the entries of repo-relative ``.bib`` files by citekey. Missing files are skipped. A key maps to every entry defining it (across files), each a dict with the entry type, defining file, the ``FIELDS_TO_SUMMARIZE`` values that were found, and the raw BibTeX. """ by_key: dict[str, list[dict[str, Any]]] = defaultdict(list) for bib_rel in bib_paths: bib_path = ROOT / bib_rel if not bib_path.exists(): continue text = bib_path.read_text(encoding="utf-8", errors="replace") starts = list(ENTRY_START_RE.finditer(text)) for i, match in enumerate(starts): start = match.start() end = starts[i + 1].start() if i + 1 < len(starts) else len(text) entry_text = text[start:end].strip() key = match.group("key") fields = { field: value for field in FIELDS_TO_SUMMARIZE if (value := extract_bib_value(entry_text, field)) is not None } by_key[key].append( { "key": key, "entry_type": match.group("entry_type"), "defined_in": bib_rel, "fields": fields, "bibtex": entry_text, } ) return by_key def slug(value: str) -> str: """Turn a unit id into a file-name stem: trim slashes, ``/`` becomes ``__``, ``.`` becomes ``_``.""" return value.strip("/").replace("/", "__").replace(".", "_") def qmd_packet_name(rel_path: str) -> str: """Return the packet file name for one QMD path, with ``/`` replaced by ``__``.""" name = rel_path.removesuffix(".qmd").replace("/", "__") return f"{name}.citation-packet.json" def audit_unit_for_path(rel_path: str, granularity: str) -> dict[str, str]: """Map a repo-relative QMD path to the audit unit its citations are grouped under. At ``qmd`` granularity every file is its own unit. Otherwise files in a ``books/vol1`` or ``books/vol2`` chapter directory group by chapter; their frontmatter, backmatter, and parts files become per-file sections (a backmatter subdirectory counts as one section, and ``_``-prefixed files are filed under ``includes/``); ``books/shared`` front and back matter group by first child; anything else is a per-file unit. Returns: A dict with ``id``, ``label``, ``root``, and ``kind``. """ if granularity == "qmd": unit_id = rel_path.removesuffix(".qmd") return { "id": unit_id, "label": rel_path, "root": rel_path, "kind": "qmd", } parts = rel_path.split("/") if parts[0] == "books" and len(parts) >= 3: if parts[1] in {"vol1", "vol2"}: volume = parts[1] section = parts[2].removesuffix(".qmd") if section in {"frontmatter", "backmatter", "parts"}: leaf = parts[-1].removesuffix(".qmd") if parts[-1].startswith("_"): leaf = f"includes/{leaf}" elif section == "backmatter" and len(parts) >= 5: leaf = parts[3] unit_id = f"{volume}/{section}/{leaf}" return { "id": unit_id, "label": unit_id, "root": rel_path, "kind": "book-section", } root = f"books/{volume}/{section}/" return { "id": f"{volume}/{section}", "label": f"{volume}/{section}", "root": root, "kind": "book-chapter", } if parts[1] == "shared" and len(parts) >= 4 and parts[2] in {"frontmatter", "backmatter"}: section = parts[2] chapter = parts[3].removesuffix(".qmd") root = f"books/shared/{section}/{parts[3]}" if not parts[3].endswith(".qmd"): root += "/" return { "id": f"book-shared/{section}/{chapter}", "label": f"book-shared/{section}/{chapter}", "root": root, "kind": "book-shared-section", } unit_id = rel_path.removesuffix(".qmd") return { "id": unit_id, "label": rel_path, "root": rel_path, "kind": "qmd", } def packet_name_for_unit(unit: dict[str, str], granularity: str) -> str: """Return a unit's packet file name: per-QMD naming at ``qmd`` granularity, else the slugged unit id.""" if granularity == "qmd": return qmd_packet_name(unit["root"]) return f"{slug(unit['id'])}.citation-packet.json" def build_packets(out_dir: Path, *, all_qmd: bool, granularity: str, max_context_chars: int) -> dict[str, Any]: """Write one citation packet JSON per audit unit and return the run summary. Existing ``*.citation-packet.json`` files in ``out_dir/packets`` are deleted first. Each occurrence is annotated with its scope, allowed bibliographies, and matching BibTeX entries (entries from the scope's bibliographies are preferred, any definition is the fallback, and the status is ``missing`` when none exists). Units without citations get no packet. Also writes ``summary.json`` and ``agent_manifest.md`` into ``out_dir``. """ all_bibs = sorted({bib for scope in SCOPES for bib in scope["bibs"]}) bib_entries = parse_bib_entries(all_bibs) packet_dir = out_dir / "packets" packet_dir.mkdir(parents=True, exist_ok=True) for stale_packet in packet_dir.glob("*.citation-packet.json"): stale_packet.unlink() grouped_qmds: dict[str, dict[str, Any]] = {} for qmd in discover_qmds(all_qmd): rel_path = rel(qmd) unit = audit_unit_for_path(rel_path, granularity) grouped_qmds.setdefault(unit["id"], {"audit_unit": unit, "qmds": []})["qmds"].append(qmd) summary_packets: list[dict[str, Any]] = [] total_occurrences = 0 total_source_files = 0 all_keys: set[str] = set() for unit_id, group in sorted(grouped_qmds.items()): unit = group["audit_unit"] citations: list[dict[str, Any]] = [] source_files: list[dict[str, Any]] = [] packet_allowed_bibs: set[str] = set() for qmd in sorted(group["qmds"]): rel_path = rel(qmd) occurrences = citation_occurrences(qmd, max_context_chars) if not occurrences: continue scope = find_scope(rel_path) allowed_bibs = set(scope["bibs"] if scope else all_bibs) packet_allowed_bibs.update(allowed_bibs) for occurrence in occurrences: key = occurrence["key"] entries = bib_entries.get(key, []) scoped_entries = [entry for entry in entries if entry["defined_in"] in allowed_bibs] selected_entries = scoped_entries or entries occurrence["source_file"] = rel_path occurrence["source_scope"] = scope["name"] if scope else "unscoped" occurrence["allowed_bibliographies"] = sorted(allowed_bibs) occurrence["reference_status"] = "resolved" if selected_entries else "missing" occurrence["reference_entries"] = selected_entries citations.append(occurrence) all_keys.add(key) source_files.append( { "path": rel_path, "scope": scope["name"] if scope else "unscoped", "citation_occurrences": len(occurrences), "unique_citekeys": sorted({item["key"] for item in occurrences}), } ) if not citations: continue packet = { "schema_version": "citation-reference-validation-packet/v1", "packet_granularity": granularity, "audit_unit": unit, "source_files": source_files, "allowed_bibliographies": sorted(packet_allowed_bibs), "citation_occurrences": len(citations), "unique_citekeys": sorted({item["key"] for item in citations}), "instructions": [ "Validate only the citation usages in this packet; this packet is one audit unit for one subagent.", "Core question: does the cited source support the local claim the citation is attached to?", "Use the embedded BibTeX first, but inspect the canonical source when metadata is not enough.", "Canonical source checks may require DOI/publisher pages, arXiv, OpenReview, official docs, project pages, or web search.", "Search-result snippets are not evidence; read enough of the source to judge the claim.", "Do not require one citation to support unrelated nearby prose.", "Do not flag broad synthesis just because the source does not use identical wording.", "Flag only high-confidence unsupported, overbroad, stale, misplaced, or source-access issues.", "Return only actionable issues plus a concise coverage summary.", ], "expected_finding_schema": { "key": "citekey", "source_file": "repo-relative qmd path", "line": "source line number", "status": "unsupported | overbroad | misplaced | stale | needs_human | missing_canonical | bib_metadata", "claim": "short paraphrase of what the local text asks the source to support", "reason": "why the source does or does not support the claim", "suggested_fix": "replace citation | add source | narrow claim | move citation | remove citation | fix metadata", "evidence": "brief source evidence, external source URL/identifier, or note that human review is needed", }, "citations": citations, } packet_path = packet_dir / packet_name_for_unit(unit, granularity) packet_path.write_text(json.dumps(packet, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") total_occurrences += len(citations) total_source_files += len(source_files) summary_packets.append( { "audit_unit": unit, "packet": display_path(packet_path), "source_files": [item["path"] for item in source_files], "citation_occurrences": len(citations), "unique_citekeys": len(packet["unique_citekeys"]), "allowed_bibliographies": sorted(packet_allowed_bibs), } ) summary = { "schema_version": "citation-reference-validation-summary/v1", "packet_granularity": granularity, "audit_units": len(summary_packets), "source_files": total_source_files, "citation_occurrences": total_occurrences, "unique_citekeys": len(all_keys), "packet_dir": display_path(packet_dir), "packets": summary_packets, } (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") write_agent_manifest(out_dir, summary) return summary def write_agent_manifest(out_dir: Path, summary: dict[str, Any]) -> None: """Write ``agent_manifest.md`` to ``out_dir``: fixed reviewer instructions and finding schema, then one bullet per packet.""" lines = [ "# Citation Reference Audit Agent Manifest", "", "Spawn one subagent per packet. Each subagent owns the whole audit unit", "listed for its packet and should not edit source files.", "", "## Subagent Task", "", "Audit the assigned chapter packet. For every citation occurrence, answer:", "does the cited source support the local claim the citation is attached to?", "", "Use the embedded BibTeX metadata first. If that is not enough, search for", "and inspect the canonical source: DOI or publisher page, arXiv, OpenReview,", "DBLP plus linked paper, official documentation, project page, standards", "document, or another primary source. Search-result snippets are not", "evidence. Read enough of the paper/source to judge the claim.", "", "Return only actionable findings plus a concise coverage summary. Do not", "flag a citation merely because it does not support unrelated nearby prose", "or because the source uses different wording. If the source cannot be", "inspected or the fit remains ambiguous after reasonable source lookup,", "return `needs_human` and say exactly what must be checked.", "", "Finding schema:", "", "```json", "{", ' "key": "citekey",', ' "source_file": "repo-relative qmd path",', ' "line": 123,', ' "status": "unsupported | overbroad | misplaced | stale | needs_human | missing_canonical | bib_metadata",', ' "claim": "short paraphrase of the local claim",', ' "reason": "why the cited source does or does not support it",', ' "suggested_fix": "replace citation | add source | narrow claim | move citation | remove citation | fix metadata",', ' "evidence": "brief source evidence and URL/identifier, or what needs human review"', "}", "```", "", "## Packets", "", ] for packet in summary["packets"]: unit = packet["audit_unit"] lines.extend( [ f"- `{packet['packet']}`", f" - audit unit: `{unit['label']}` ({unit['kind']})", f" - citations: {packet['citation_occurrences']}", f" - source files: {len(packet['source_files'])}", f" - bibliographies: {', '.join(f'`{bib}`' for bib in packet['allowed_bibliographies'])}", ] ) (out_dir / "agent_manifest.md").write_text("\n".join(lines) + "\n", encoding="utf-8") def main() -> int: """Build packets under ``--out-dir`` (relative paths resolve against the repo root), print a summary line, and return 0.""" parser = argparse.ArgumentParser(description="Build citation-reference validation packets.") parser.add_argument( "--out-dir", default="review/citation-reference-validation", help="Output directory, relative to repository root or absolute.", ) parser.add_argument( "--all-qmd", action="store_true", help="Scan every .qmd in the repository instead of the scoped bibliography trees.", ) parser.add_argument( "--granularity", choices=("chapter", "qmd"), default="chapter", help="Packet sharding strategy. Default: chapter.", ) parser.add_argument( "--max-context-chars", type=int, default=1800, help="Maximum paragraph context characters per citation occurrence.", ) args = parser.parse_args() out_dir = Path(args.out_dir) if not out_dir.is_absolute(): out_dir = ROOT / out_dir out_dir.mkdir(parents=True, exist_ok=True) summary = build_packets( out_dir, all_qmd=args.all_qmd, granularity=args.granularity, max_context_chars=args.max_context_chars, ) print( "Wrote " f"{summary['audit_units']} {summary['packet_granularity']} packets covering " f"{summary['source_files']} source files with " f"{summary['citation_occurrences']} citation occurrences to " f"{Path(summary['packet_dir'])}" ) return 0 if __name__ == "__main__": raise SystemExit(main())