""" Native validation commands for MLSysBook Binder CLI. Every `book-*` pre-commit hook dispatches through `./book/binder check ` so there is one entry point, one error format, and one place to add new checks. See the `ValidateCommand` class docstring below for the contract: 1. Add a `_run_` method that returns a ValidationRunResult. 2. Add a `Scope("", "_run_", default=...)` to the relevant group in GROUPS. Use `default=False` if the scope still fails on dev or is intentionally opt-in (slow / heavy / manual stage). 3. Surface any new flags on the argparse block in `run()`. Pre-commit picks the new scope up automatically once `default=True` — no YAML edit needed. Ad-hoc audits use `--all-scopes` or `--scope `. Some checks still delegate to scripts under book/tools/scripts/ (tables, spelling, epub, sources, grid-tables, images) and to standalone audit scripts under book/tools/audit/index/. Prefer new logic in cli/checks/ with normal imports; see book/cli/README.md "Check implementation layout". Legacy `_delegate_script` / importlib paths are being phased out. """ from __future__ import annotations import argparse import json import os import re import subprocess import sys import time from collections import defaultdict from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple from rich.console import Console from rich.markup import escape as _rich_escape from rich.panel import Panel from rich.table import Table from . import reference_check console = Console() @dataclass class ValidationIssue: file: str line: int code: str message: str severity: str = "error" context: str = "" suggestion: str = "" def to_dict(self) -> Dict[str, Any]: payload = { "file": self.file, "line": self.line, "code": self.code, "message": self.message, "severity": self.severity, "context": self.context, } if self.suggestion: payload["suggestion"] = self.suggestion return payload @dataclass class ValidationRunResult: name: str description: str files_checked: int issues: List[ValidationIssue] elapsed_ms: int @property def passed(self) -> bool: return len(self.issues) == 0 def to_dict(self) -> Dict[str, Any]: return { "name": self.name, "description": self.description, "files_checked": self.files_checked, "passed": self.passed, "issue_count": len(self.issues), "elapsed_ms": self.elapsed_ms, "issues": [issue.to_dict() for issue in self.issues], } INLINE_REF_PATTERN = re.compile(r"`\{python\}\s+(\w+(?:\.\w+)?)`") INLINE_EXPR_PATTERN = re.compile(r"`\{python\}\s+([^`]+?)`") CELL_START_PATTERN = re.compile(r"^```\{python\}|^```python") CELL_END_PATTERN = re.compile(r"^```\s*$") ASSIGN_PATTERN = re.compile(r"^([A-Za-z_]\w*)\s*=") # Tuple unpacking: "a, b = ..." — captures all names on the left side TUPLE_ASSIGN_PATTERN = re.compile(r"^((?:[A-Za-z_]\w*\s*,\s*)+[A-Za-z_]\w*)\s*=") CLASS_DEF_PATTERN = re.compile(r"^class\s+(\w+)\s*[:(]") GRID_TABLE_SEP_PATTERN = re.compile(r"^\+[-:=+]+\+$") # NOTE: The old LATEX_INLINE_PATTERN regex could treat the closing `$` of one # math span as the opening `$` of the next, producing false positives on lines # such as `$P_0$: `{python} x_str`. $P_t$`. Use a scanner so only actual # `$...$` and `\(...\)` spans are checked for inline Python. def _is_escaped(text: str, idx: int) -> bool: backslashes = 0 j = idx - 1 while j >= 0 and text[j] == "\\": backslashes += 1 j -= 1 return backslashes % 2 == 1 def _inline_math_spans(text: str) -> List[str]: """Return inline math spans from one Markdown source line.""" spans: List[str] = [] i = 0 n = len(text) while i < n: if text.startswith(r"\(", i) and not _is_escaped(text, i): end = text.find(r"\)", i + 2) if end == -1: i += 2 continue spans.append(text[i:end + 2]) i = end + 2 continue if text[i] == "$" and not _is_escaped(text, i): if (i + 1 < n and text[i + 1] == "$") or (i > 0 and text[i - 1] == "$"): i += 1 continue j = i + 1 while j < n: if text[j] == "$" and not _is_escaped(text, j): if (j + 1 < n and text[j + 1] == "$") or text[j - 1] == "$": j += 1 continue spans.append(text[i:j + 1]) i = j + 1 break j += 1 else: i += 1 continue i += 1 return spans def _inline_python_math_spans(text: str) -> List[str]: return [span for span in _inline_math_spans(text) if "{python}" in span] LATEX_OPERATOR_AFTER_PATTERN = re.compile( r"\s*\$\\(times|approx|ll|gg|mu|le|ge|neq|pm|cdot|div)" ) CANONICAL_INLINE_SUFFIXES = ("_str", "_math", "_eq", "_frac") def _inline_python_latex_operator_warnings(text: str) -> List[str]: """Refs adjacent to LaTeX operators without a canonical display suffix.""" refs: List[str] = [] for match in INLINE_EXPR_PATTERN.finditer(text): after = text[match.end():] if not LATEX_OPERATOR_AFTER_PATTERN.match(after): continue expr = match.group(1).strip() name = expr.split("[", 1)[0].split("(", 1)[0].strip() final = name.split(".")[-1] if not final.endswith(CANONICAL_INLINE_SUFFIXES): refs.append(match.group(0)) return refs # LATEX_ADJACENT_PATTERN checks were retired from validate_inline_refs.py along # with mlsysim.fmt's MarkdownStr migration. They had been guarding against # Quarto's auto-escape silently corrupting commas and decimals inside $..$ math # mode — a bug class that no longer exists now that fmt() returns a Markdown- # rendering string that bypasses the escape. See mlsysim/mlsysim/fmt.py and # the project math rules. CITATION_REF_PATTERN = re.compile(r"@([A-Za-z0-9_:\-.]+)") CITATION_BRACKET_PATTERN = re.compile(r"\[-?@[A-Za-z0-9_:\-.]+(?:;\s*-?@[A-Za-z0-9_:\-.]+)*\]") LABEL_DEF_PATTERNS = { "Figure": [ re.compile(r"\{#(fig-[\w-]+)"), # {#fig-xyz ...} re.compile(r"#\|\s*label:\s*(fig-[\w-]+)"), # #| label: fig-xyz re.compile(r"%%\|\s*label:\s*(fig-[\w-]+)"), # %%| label: fig-xyz (Jupyter) ], "Table": [ re.compile(r"\{#(tbl-[\w-]+)"), # {#tbl-xyz} re.compile(r"#\|\s*label:\s*(tbl-[\w-]+)"), # #| label: tbl-xyz ], "Section": [ re.compile(r"\{#(sec-[\w-]+)"), # {#sec-xyz} re.compile(r"^id:\s*(sec-[\w-]+)"), # YAML id: sec-xyz ], "Equation": [re.compile(r"\{#(eq-[\w-]+)")], # {#eq-xyz} "Listing": [ re.compile(r"\{#(lst-[\w-]+)"), # {#lst-xyz ...} re.compile(r"#\|\s*label:\s*(lst-[\w-]+)"), # #| label: lst-xyz ], # `alg-` is Quarto's native algorithm float; `algo-` is the # mlsysbook-ext/pseudocode extension's prefix (native @alg- collides with # Quarto's algorithm theorem env and FATALs the build, so the book uses # @algo-). Accept both, including the pseudocode chunk-option label form. "Algorithm": [ re.compile(r"\{#(algo?-[\w-]+)"), # {#alg-xyz} / {#algo-xyz} re.compile(r"#\|\s*label:\s*(algo?-[\w-]+)"), # #| label: algo-xyz (pseudocode ext) ], } LABEL_REF_PATTERN = re.compile(r"@((?:[Ff]ig|[Tt]bl|[Ss]ec|[Ee]q|[Ll]st|[Aa]lgo|[Aa]lg)-[\w-]+)") # Pseudocode algorithm label declared as a chunk option inside a ```pseudocode # fence (`#| label: algo-xyz`). Harvested even inside code fences so @algo- # references resolve (see _run_unreferenced_labels). Accepts alg-/algo-. PSEUDOCODE_LABEL_PATTERN = re.compile(r"#\|\s*label:\s*(algo?-[\w-]+)") EXCLUDED_CITATION_PREFIXES = ( "fig-", "tbl-", "sec-", "eq-", "lst-", "alg-", "algo-", "ch-", "nb-", "Fig-", "Tbl-", "Sec-", "Eq-", "Lst-", "Alg-", "Algo-", ) # Captionless float baseline: per-file counts of pre-existing violations # grandfathered when the caption-required / label-required scopes landed. # A commit that increases any per-file count fails the check; a commit that # *decreases* a count is fine (debt going down). Regenerate with: # ./book/binder check tables --scope caption-required --update-baseline CAPTIONS_BASELINE_PATH = ( Path(__file__).resolve().parent.parent.parent / "tools" / "audit" / "baselines" / "captions_baseline.json" ) # Existing bibliography citations inside scaffolding blocks. New citations in # Purpose, learning objectives, or takeaways fail `refs.scaffold-citations`. SCAFFOLD_CITATION_BASELINE_PATH = ( Path(__file__).resolve().parent.parent.parent / "tools" / "audit" / "baselines" / "scaffold_citation_baseline.json" ) @dataclass(frozen=True) class Scope: """One scope inside a binder check group. A "scope" is a single check that lives inside a group (e.g. `refs.citations`, `prose.contractions`). Scopes are the unit of dispatch: `./binder check refs --scope citations` runs one. `./binder check refs` runs every scope flagged `default=True`. Pre-commit consumes the `default=True` set; ad-hoc audits opt into everything via `./binder check refs --all-scopes`. Why this matters: as the corpus matures, new scopes get added in a "warning-only / not-yet-clean" state. Marking them `default=False` lets them ship without blocking commits, while still being runnable on demand. Flip to `default=True` once dev is clean for that scope. """ name: str runner: str default: bool = True note: str = "" class ValidateCommand: """Native `binder check` command group (also available as `binder validate`). The hierarchy is: binder check # all default=True scopes binder check --scope # one specific scope binder check --all-scopes # every scope (incl. default=False) The `default` flag on each Scope encodes "is this scope curated for the pre-commit/CI baseline today." That is the *only* concept the YAML and CI need to know about. Adding a new check is therefore three things: 1. Add a `_run_` method below that returns a ValidationRunResult. 2. Add a `Scope("", "_run_", default=...)` entry to the appropriate group below. Use `default=False` when the scope still fails on dev or is intentionally opt-in (slow, heavy, manual). 3. If the scope needs new flags, add them to the argparse block in `run()` and dispatch them in `_run_group`. Structured fix output (REQUIRED for every ValidationIssue). The check output is consumed by automated fixers via `binder check --json`, so each issue must be machine-fixable on its own: - `file` + `line` — exact location. - `code` — a stable snake_case identifier (never reword it; tools key on it). - `suggestion` — the EXACT fix as `""`. Always set it when the fix is deterministic. When it needs human judgment (e.g. a prose range), still give the best concrete form and say so in `message`; never leave callers with only "fix this". - `context` — the surrounding text so a fixer can locate the span within the line. The human-readable text and `--json` both render from the same ValidationIssue, so populating `suggestion` serves both. No new pre-commit hook needed. Pre-commit runs `./binder check ` and picks up the new scope automatically once it is `default=True`. Group catalogue: cli — public Binder command contract (help/error shape) refs — refs / citations / cross-refs hygiene labels — duplicate labels, orphaned/unreferenced labels headers — section header IDs and headline-case policy bib — bibliography (.bib) hygiene + canonical forms footnotes — definition-shape, placement, integrity, cross-chapter, capitalization figures — captions/alt-text, float flow, image files markup — low-level markup (patterns, div fences, dropcaps) prose — prose style (contractions, dup words, ASCII, ...) punctuation — em-dash, slash, vs., e.g./i.e., en-dash ranges numbers — units / percent / binary units / currency notation math — \\times spacing, attr-leaks, render-audit (manual) structure — heading levels, H2 landings, parts, Purpose-unnumbered code — python-echo, _str/_math LaTeX leak, LEGO dead code tables — grid → pipe, table content index — \\index{} placement, anti-patterns, tag-placement, xrefs images — file formats, externals, SVG well-formedness json — JSON syntax units — physics-engine unit conversion tests notation — iron-law symbol consistency (BW, R_peak, L_lat, D_vol) spelling — prose + TikZ spell checks (requires aspell) epub — source hygiene, smoke checks, epubcheck (built) pdf — post-build PDF cross-ref / leak scan (built artifact) registry — constants → registry migration gates sources — source citation patterns references — bibliography vs. academic DBs (hallucinator; slow) content — content tree (shared/, frontmatter/ required) all — every group above """ # Single source of truth for the group → scopes hierarchy. # Order within each group: cheaper / earlier checks first, since later # scopes typically build on earlier ones (e.g. labels.duplicates before # labels.orphans). default=False scopes are listed alongside their kin # so the file reads as a complete map of what *could* run. GROUPS: Dict[str, List[Scope]] = { "cli": [ Scope("contract", "_run_cli_contract", note="public Binder command contract"), Scope("binder-canonical", "_run_binder_canonical", note="book-content pre-commit hooks must dispatch through ./book/binder", default=True), ], "refs": [ # python-syntax: passes on dev but never wired to pre-commit historically; # leaving default=False to preserve current coverage exactly. Flip later # if we want it in the curated set. Scope("python-syntax", "_run_python_syntax", default=False), # inline-python + self-ref currently fail on dev (corpus debt). # Runnable on demand via --scope or --all-scopes. Scope("inline-python", "_run_inline_python", default=False), Scope("cross-refs", "_run_refs"), Scope("citations", "_run_citations"), Scope("scaffold-citations", "_run_scaffold_citations", note="no bibliography citations in Purpose, learning objectives, or takeaways; current debt baseline-ratcheted"), Scope("duplicate-year", "_run_duplicate_citation_year", note='"[@foo1964] (1964)" — redundant year after citation'), Scope("duplicate-key", "_run_duplicate_citation_key", note='"[@k; @k]" or "[@k] [@k]" — same-key citeproc dup'), Scope("manual-bracket", "_run_manual_bracket_citation", note='*et al.*(…) — manual attribution next to [@k]'), Scope("principles", "_run_principle_refs", note="principles use #pri-* IDs and Principle \\ref{pri-*}"), # inline scope is run with --check-patterns OFF in the curated set # (matches former pre-commit hook). Pattern hazards still live in # the corpus; opt in with --check-patterns when cleaning them up. Scope("inline", "_run_inline_refs"), Scope("self-ref", "_run_self_referential", default=False), Scope("capitalized", "_run_mitpress_capitalized_refs", note='"chapter 12" lowercase in prose (§10.3.2)'), Scope("xref-case", "_run_xref_sentence_start_case", note="@-prefix casing must match sentence position: @Fig- at a sentence start, @fig- mid-sentence (both directions; sec/fig/tbl/eq/lst/alg)"), ], "labels": [ # duplicates and orphans are both curated, but each carries its # own implicit filter when invoked from pre-commit: # - duplicates: figures + tables + listings only (sections / # equations have legitimate dup-key collisions in vol2 WIP) # - orphans: vol1 only (vol2 has many forward labels for # not-yet-authored chapters) # The YAML hook embeds the appropriate flags. fig-labels currently # fails (corpus debt) so it stays default=False until cleaned up. Scope("duplicates", "_run_duplicate_labels"), Scope("orphans", "_run_unreferenced_labels"), Scope("fig-labels", "_run_fig_label_underscores", default=False), ], "headers": [ # ids: vol1 only (vol2 chapters are early-development; many # sections still missing IDs). YAML hook embeds --vol1. Scope("ids", "_run_headers"), Scope("case", "_run_mitpress_heading_case", note="MIT Press headline-case (§10.3.1)"), ], "bib": [ Scope("hygiene", "_run_bib_hygiene"), Scope("style", "_run_bib_style", note="baseline-ratcheted metadata style: title case, author initials, venues, publishers"), Scope("integrity", "_run_bib_integrity", note="volume-scoped citation resolution: no missing keys or cross-volume leaks"), Scope("orphans", "_run_bib_orphans", default=False, note="unused entries in volume BibTeX files; opt-in because current bibs retain legacy/canonical background"), # key-content fails on dev (legacy keys grandfathered). # Was never wired to pre-commit; default=False preserves status quo. Scope("key-content", "_run_bib_key_content", default=False), # key-year: ERRORS-ONLY subset of key-content (key year vs entry year, # >1yr gap = silent citation corruption). Gated at pre-commit; the # surname WARNINGS (intentional project-name keys) stay opt-in above. Scope("key-year", "_run_bib_key_year", note="key year must match entry year (errors-only; pre-commit gate)"), # duplicate-keys: two entries sharing a citekey -> ambiguous citation. Scope("duplicate-keys", "_run_bib_duplicate_keys", note="every citekey must be unique across the bibliography"), # links: every doi/url field must resolve (MIT Press "confirm all # URLs"). Network-bound -> opt-in, not a pre-commit gate. Scope("links", "_run_bib_links", default=False, note="DOI/URL liveness: every locator resolves (network; opt-in)"), ], "footnotes": [ Scope( "definition-shape", "_run_footnote_definition_shape", note="bold head + colon per book-prose §5 (shapes S1–S5)", ), Scope("placement", "_run_footnote_placement"), Scope("integrity", "_run_footnote_refs"), Scope("cross-chapter", "_run_footnote_cross_chapter", default=False), Scope("capitalization", "_run_footnote_capitalization"), ], "figures": [ Scope("captions", "_run_figures"), Scope("caption-heads", "_run_mitpress_caption_head_style", note="fig-cap/tbl-cap/lst-cap use **Bold Title**: Explanation"), Scope("div-syntax", "_run_figure_div_syntax"), Scope("label-required", "_run_figure_label_required", note="![](...) markdown images in body prose need {#fig-X}"), # flow: deferred to copyeditor's PDF layout pass (2026-05-03). # The "near first reference" heuristic conflicts with the # repositioning the copyeditor does to balance pages in print. # Re-enable when we own figure placement again. Scope("flow", "_run_float_flow", default=False), Scope("files", "_run_images"), Scope("alt-text-style", "_run_mitpress_alt_text_style", note="alt-text follows body-prose rules (§10.12)"), ], # ------------------------------------------------------------------ # Semantic check groups: classify checks by WHAT is validated # (markup patterns, prose style, punctuation, numbers, math, etc.), # not by where-the-rule-came-from. A rule's provenance belongs in # a comment, not a command name — hence no `mitpress-` prefix on # scopes. See the project prose style guide for style provenance. # ------------------------------------------------------------------ "markup": [ Scope("patterns", "_run_rendering", note="low-level markup (backticks, dollar signs, asterisks)"), Scope("list-spacing", "_run_markdown_list_spacing", note="blank line required between bold lead-in paragraphs and lists"), Scope("div-fences", "_run_div_fences", note=":::/ :::: balance and form"), Scope("callouts", "_run_callout_structure", note="supported callout types, titles, and attributes"), # default=False until vol2 narrative callouts are normalized to the # same schema vol1 uses; flip to True once `--scope callout-schema` # is clean on dev for both volumes. Scope("callout-schema", "_run_callout_schema", default=False, note="per-type bold-label structure (war-story = Context/Failure mode/Systems lesson)"), Scope("dropcaps", "_run_dropcaps"), ], "prose": [ Scope("contractions", "_run_mitpress_contractions", note='no "can\'t", "it\'s" in body prose'), Scope("spelling-dict", "_run_mitpress_spelling_dict", note="canonical spellings (§10.7): trade-off, dataset, data center, Wi-Fi, …"), Scope("duplicate-words", "_run_duplicate_words"), Scope("unblended-prose", "_run_unblended_prose", note="space after period"), Scope("above-below", "_run_mitpress_above_below", note='no "above"/"below" spatial refs'), # ascii: corpus has many legitimate non-ASCII chars (em-dashes, # smart quotes from imports). Default=False until corpus-wide # ASCII pass lands. Scope("ascii", "_run_ascii", default=False), Scope("acknowledgements", "_run_mitpress_acknowledgements", note="American spelling: Acknowledgments"), Scope("compound-prefix", "_run_mitpress_compound_prefix", note="pre-/non- close-up (§10.8)"), Scope("concept-caps", "_run_mitpress_concept_term_capitalization", note="iron law, memory wall lowercase (§10.3)"), # abbreviation-first-use currently opt-in because some findings # require editorial judgment. Scope("abbreviation-first-use", "_run_mitpress_abbreviation_first_use", default=False), Scope("latin-abbrevs", "_run_mitpress_latin_running_text", note="viz./e.g./etc. in running text (§10.6)"), ], "punctuation": [ Scope("emdash", "_run_mitpress_spaced_emdash", note="word—word, no spaces"), Scope("slash", "_run_mitpress_spaced_slash", note="training/inference, no spaces"), Scope("vs-period", "_run_mitpress_vs_period", note="vs. not vs"), Scope("eg-ie-comma", "_run_mitpress_eg_ie_comma", note="comma after e.g./i.e."), Scope("hyphen-range", "_run_mitpress_hyphen_range", note="en-dash for number ranges"), ], "numbers": [ Scope("unit-spacing", "_run_unit_spacing", note="100 ms, not 100ms"), Scope("binary-units", "_run_binary_units", note="GB/TB not GiB/TiB"), Scope("percent-spacing", "_run_percent_spacing", note="no space before %"), Scope("percent-word-before", "_run_percent_word_before", note="digits before 'percent', not spelled-out words (5 percent not five percent)"), Scope("percent-in-captions", "_run_mitpress_percent_in_captions", note="spell out 'percent' in captions"), Scope("percent-in-tables", "_run_mitpress_percent_in_tables", note="use % symbol inside tables, not the word 'percent'"), Scope("percent-in-prose", "_run_mitpress_percent_in_prose", note="spell out 'percent' in body prose, not the % symbol"), Scope("currency", "_run_currency_style", note="use $ in content; USD defined once in notation"), Scope("rendered-currency", "_run_rendered_currency_style", note="post-render USD/$$/raw-LaTeX currency artifacts", default=False), ], "math": [ Scope("times-spacing", "_run_times_spacing", note="space after $\\times$"), Scope("times-product-spacing", "_run_times_product_spacing", note="space before $\\times$ after inline code"), Scope("attr-leaks", "_run_attr_latex_leaks", note="LaTeX in title=/fig-cap/tbl-cap/fig-alt/tbl-alt"), Scope("canonical", "_run_math_canonical", note="fmt-family + _str/_math/_eq/_frac suffix discipline (LEGO)"), Scope("prose-contract", "_run_fmt_prose_contract", note="formatter-owned glyphs/units are not duplicated in prose"), # Added 2026-05-26: blocklist for banned suffix= values in fmt() # calls (wrong unit conventions like TFLOPS, Gbps, etc.). Scope("suffix-consistency", "_run_suffix_consistency", note='banned suffix= values (TFLOPS, Gbps, etc.)'), Scope("multiplier-style", "_run_math_multiplier_style", note="body-prose multiplier suffixes, Unicode ×, product spacing", default=False), # Typed-formatter migration gate: value-kind (percent/multiplier/ # percentage-points/count-scale) must be a typed formatter, not a # free-text suffix= on fmt()/fmt_int(). default=False until the # corpus migration completes; flip to gate on pre-commit. Scope("suffix-semantics", "_run_fmt_semantic_suffix", note="value-kind in suffix= must be a typed formatter", default=False), # render-audit builds every chapter (~10 min). Manual stage only; # default=False ensures `binder check math` stays under 1s. Scope("render-audit", "_run_math_render_audit", default=False), ], "structure": [ Scope("heading-levels", "_run_heading_levels", note="H1→H2→H3 hierarchy"), Scope("h2-landings", "_run_h2_landings", note="## sections need prose before ### subsections"), Scope("parts", "_run_parts", note="part keys valid"), Scope("purpose-unnumbered", "_run_purpose_unnumbered", note="Purpose sections unnumbered"), ], "code": [ Scope("python-echo", "_run_python_echo", note="echo: false on python blocks"), Scope("str-latex-leak", "_run_str_latex_leak", note="*_str exports must not contain raw LaTeX"), Scope("lego-dead-code", "_run_lego_dead_code", note="LEGO variables defined but never referenced"), Scope("lego-prose-literals", "_run_lego_prose_literals", note="hardcoded numbers in callout walkthroughs that share {python} refs"), Scope("lego-prose-units", "_run_lego_prose_units", note="unit/currency tokens after {python} *_str refs", default=False), Scope("lego-load-pint", "_run_lego_load_pint", note="physical *_value assignments must use ureg/registry", default=False), Scope("lego-equations", "_run_lego_equations", note="A/B=C prose lines must match computed values", default=False), Scope("lego-units", "_run_lego_units", note="LEGO unit discipline (warning-only + baseline)", default=True), # Added 2026-05-26: \\${python} collision — escaped dollar before # {python} silently fails to render; correct form is \\$\\`{python}. Scope("python-dollar-collision", "_run_python_dollar_collision", note="\\${python} collision — escaped dollar silently kills inline expr"), # Added 2026-05-26: scan built HTML for literal {python} text that # leaked through Quarto (expression failed to evaluate). Requires # a prior html-audit build. Default=False (opt-in audit). Scope("rendered-python-leak", "_run_rendered_python_leak", note="literal {python} leaked into rendered HTML", default=False), ], "tables": [ Scope("grid-tables", "_run_grid_tables", note="prefer pipe tables"), Scope("content", "_run_table_content", note="bare pipes, fracs, HTML entities"), Scope("caption-required", "_run_table_caption_required", note="body-prose pipe tables need : caption {#tbl-X}"), Scope("caption-position", "_run_table_caption_position", note="pipe-table captions belong below the table"), Scope("caption-orphan", "_run_table_caption_orphan", note="caption between two `:::` closes breaks EPUB"), Scope("caption-detached", "_run_table_caption_detached", note="prose between pipe table and its `: caption {#tbl-}` breaks xref"), ], "listings": [ # Listings in this book always use ::: {#lst-X lst-cap="..."} divs. # caption-required enforces that any #lst- label carries an # lst-cap (matching the figure/table caption-head convention). Scope("caption-required", "_run_listing_caption_required", note="any #lst- listing must carry lst-cap=..."), ], "index": [ Scope("placement", "_run_indexes", note='\\index{} not inline with headings/callouts'), # Migrated 2026-05-06: was book/tools/audit/index/check_anti_patterns.py Scope("anti-patterns", "_run_index_anti_patterns", note="anti-patterns from the project index rules §9"), # Migrated 2026-05-06: was book/tools/audit/index/check_tag_placement.py Scope("tag-placement", "_run_index_tag_placement", note='\\index{} not inside **bold**, *italic*, `code`, or headings'), # Migrated 2026-05-06: was book/tools/audit/index/check_xref_resolves.py Scope("xref-resolves", "_run_index_xref_resolves", note="every |see / |seealso target resolves to a real main entry"), # Added 2026-05-17: catches \index{} inside Python code fences, # $..$ / $$..$$ math, and fig-cap/fig-alt/tbl-cap/lst-cap/title= # attribute strings. Complements `placement` (which checks # heading/div/footnote adjacency). Scope("forbidden-contexts", "_run_index_placement_contexts", note='\\index{} not inside code / math / attribute strings'), ], "images": [ Scope("formats", "_run_image_formats"), Scope("external", "_run_external_images"), Scope("svg-xml", "_run_svg_wellformedness"), ], "json": [ Scope("syntax", "_run_json_syntax"), ], "units": [ # Physics-engine unit tests. Heavyweight for pre-commit; runs in # under 5s today but is fundamentally a test suite, not a static # check. TODO: consider moving to CI. Scope("physics", "_run_unit_tests"), ], "notation": [ Scope("consistency", "_run_notation_consistency", note="iron-law symbols (BW, R_peak, L_lat, D_vol)"), ], "spelling": [ # Both scopes require aspell. Heavy and not part of pre-commit # today. Default=False until aspell is on every contributor box. Scope("prose", "_run_spelling_prose", default=False), Scope("tikz", "_run_spelling_tikz", default=False), ], "epub": [ # Fast source-level invariants (<1s across SVGs + .bib). # Suitable for pre-commit. Scope("hygiene", "_run_epub_hygiene"), # Reader-compatibility smoke checks against the built EPUB. # Requires a prior epub build under _build/epub-vol*/. # Default=False because it presupposes a build step. Scope("smoke", "_run_epub_smoke", default=False), # Full W3C epubcheck validation. Requires epubcheck + JRE, # slow (~30s per volume). CI-only. Scope("epubcheck", "_run_epubcheck", default=False), ], "pdf": [ # Scans _build/pdf-vol*/ artifact via pdftotext. Requires # --vol1 or --vol2 (or run both explicitly). Scope("verify", "_run_pdf_verify"), # Narrow rendered-PDF counter regression gate. Useful after # chapter-only builds, where full cross-reference verification can # be noisy because other chapters are intentionally absent. Scope("numbering", "_run_pdf_numbering"), # Narrow rendered-layout gate for table/prose crowding. Useful # after chapter-only builds where `verify` can be noisy. Scope("table-spacing", "_run_pdf_table_spacing"), # Added 2026-05-26: scan PDF text for "UserWarning" strings that # may indicate a Python warning leaked into the rendered output. # Default=False — post-build audit, not a pre-commit gate. Scope("pdf-warnings", "_run_pdf_warnings", default=False), ], "registry": [ Scope("sources", "_run_registry_sources", note="QMD LEGO cells: banned legacy constant/registry imports"), Scope("tests", "_run_registry_tests", note="mlsysim pytest registry gate tests"), Scope("appendix", "_run_registry_appendix", note="appendix LEGO cells exec against live registry"), Scope("anchors", "_run_registry_anchors", note="paper anchor validation"), Scope("yaml-pending", "_run_registry_yaml_pending", note="audit YAML should_change=true count must be zero"), ], "sources": [ Scope("citations", "_run_sources"), ], "references": [ # Hits external academic DBs (slow, network-dependent). # Manual / CI-only by design. Scope("hallucinator", "_run_check_references", default=False), ], "content": [ # Release-only structure check for the canonical two-volume tree. Scope("tree", "_run_content_tree", default=False), ], } def __init__(self, config_manager, chapter_discovery): self.config_manager = config_manager self.chapter_discovery = chapter_discovery def run(self, args: List[str]) -> bool: if args == ["help"]: self._print_check_help() return True # Per-group help: `./binder check help` prints a # dedicated help panel for that group, including concrete # error codes. This lives above argparse because `help` is # not a valid scope in the GROUPS dict; intercepting it here # keeps the argparse surface narrow while still giving each # group room for bespoke guidance. if len(args) == 2 and args[1] == "help": group = args[0] if group in self.GROUPS: self._print_group_help(group) return True all_group_names = list(self.GROUPS.keys()) + ["all"] parser = argparse.ArgumentParser( prog="binder check", description="Run quality checks on book content", add_help=True, ) parser.add_argument( "subcommand", nargs="?", choices=all_group_names, help="Check group to run; use `binder check` to see the live catalogue", ) parser.add_argument( "files", nargs="*", help="Optional file(s) or directories to check; used by pre-commit", ) parser.add_argument("--scope", default=None, help="Narrow to a specific check within a group") parser.add_argument( "--all-scopes", action="store_true", default=False, help="Include scopes marked default=False (heavy / not-yet-clean / opt-in). " "Without this flag, `binder check ` runs only the curated set.", ) parser.add_argument("--path", default=None, help="File or directory path to check") parser.add_argument("--vol1", action="store_true", help="Scope to Volume I") parser.add_argument("--vol2", action="store_true", help="Scope to Volume II") parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON output") parser.add_argument("--verbose", "-v", action="store_true", default=True, help="Show context for each issue (default)") parser.add_argument("--quiet", "-q", action="store_true", dest="quiet", help="Suppress verbose output") parser.add_argument("--citations-in-code", action="store_true", help="refs: check citations in code fences") parser.add_argument("--citations-in-raw", action="store_true", help="refs: check citations in raw blocks") # Default OFF: the inline scope ships in the curated set without # pattern checks (matches pre-2026-05 hook behavior; the pattern # corpus has known noise that needs cleanup before flipping default). parser.add_argument("--check-patterns", action="store_true", default=False, help="refs --scope inline: include pattern hazard checks (opt-in; noisy on dev)") parser.add_argument("--no-check-patterns", action="store_false", dest="check_patterns", help="refs --scope inline: skip pattern hazard checks (default)") parser.add_argument("--check-scope", action="store_true", default=False, help="refs --scope inline: detect bare variable refs in class bodies that need ClassName.attr") parser.add_argument("--no-check-scope", action="store_false", dest="check_scope", help="refs --scope inline: skip scope analysis") parser.add_argument("--include-lightbox", action="store_true", default=False, help="math --scope attr-leaks: also surface fig-cap/tbl-cap math that leaks into HTML lightbox tooltips (warning, opt-in; ~70 pre-existing instances on dev)") parser.add_argument("--figures", action="store_true", help="labels: filter to figures") parser.add_argument("--tables", action="store_true", help="labels: filter to tables") parser.add_argument("--sections", action="store_true", help="labels: filter to sections") parser.add_argument("--equations", action="store_true", help="labels: filter to equations") parser.add_argument("--listings", action="store_true", help="labels: filter to listings") parser.add_argument("--all-types", action="store_true", help="labels: all label types") parser.add_argument("-f", "--file", dest="refs_file", action="append", metavar="BIB", help="references: .bib file(s) to check") parser.add_argument("-o", "--output", dest="refs_output", metavar="FILE", help="references: write report to FILE") parser.add_argument("--limit", type=int, dest="refs_limit", metavar="N", help="references: check only first N refs (quick test)") parser.add_argument("--skip-verified", dest="refs_skip_verified", action="store_true", help="references: skip refs already verified in cache") parser.add_argument("--thorough", dest="refs_thorough", action="store_true", help="references: revalidate all refs (ignore cache)") parser.add_argument("--refs-cache", dest="refs_cache", metavar="FILE", help="references: cache file (default: .references_verified.json in repo root)") parser.add_argument("--only-from-report", dest="refs_only_from_report", metavar="FILE", help="references: validate only keys that had issues in this report file") parser.add_argument("--only-keys", dest="refs_only_keys_file", metavar="FILE", help="references: validate only keys listed in FILE (one key per line)") # epub --scope hygiene: auto-repair source files in-place. parser.add_argument( "--fix", action="store_true", help="epub --scope hygiene: auto-repair SVG / BibTeX source invariants in place", ) # epub --scope epubcheck: thresholds for FATAL and ERROR counts. # Defaults: MAX_FATAL=0 (any FATAL fails the check — Kindle / # ClearView reject), MAX_ERRORS=unlimited (grandfathered while the # RSC-005/RSC-012 baselines stabilize). Tighten with --max-errors 0 # once the baseline is clean. parser.add_argument( "--max-fatal", type=int, default=0, help="epub --scope epubcheck: max FATAL issues allowed before failure (default: 0)", ) parser.add_argument( "--max-errors", type=int, default=None, help="epub --scope epubcheck: max ERROR issues allowed before failure (default: unlimited)", ) # Ratchet: fail only when counts *increase* over a recorded baseline. # Lets CI block regression without cliff-failing during incremental # cleanup. Pairs with --update-baseline to re-record after a fix. parser.add_argument( "--baseline", metavar="PATH", default=None, help="epub --scope epubcheck: JSON baseline of per-volume counts; fail only if current counts exceed baseline", ) parser.add_argument( "--update-baseline", action="store_true", help="epub --scope epubcheck: rewrite --baseline file with current counts (requires --baseline)", ) parser.add_argument( "--log", dest="pdf_log", type=str, default=None, help="pdf --scope verify: optional Quarto render log to scan for crossref warnings", ) try: ns = parser.parse_args(args) except SystemExit: # argparse uses SystemExit(0) for --help and non-zero for parse errors. return ("-h" in args) or ("--help" in args) if not ns.subcommand: self._print_check_help() return True root_path = self._resolve_path(ns.path, ns.vol1, ns.vol2) if not root_path.exists(): self._emit(ns.json, {"status": "error", "message": f"Path not found: {root_path}"}, failed=True) return False runs: List[ValidationRunResult] = [] if ns.subcommand == "all": for group_name in self.GROUPS: runs.extend(self._run_group(group_name, None, root_path, ns)) else: group_name = ns.subcommand scope = ns.scope if scope and not any(sc.name == scope for sc in self.GROUPS.get(group_name, [])): valid = [sc.name for sc in self.GROUPS[group_name]] console.print(f"[red]Unknown scope '{scope}' for group '{group_name}'.[/red]") console.print(f"[yellow]Valid scopes: {', '.join(valid)}[/yellow]") return False runs.extend(self._run_group(group_name, scope, root_path, ns)) any_failed = any(not run.passed for run in runs) summary = { "status": "failed" if any_failed else "passed", "command": ns.subcommand, "path": str(root_path), "runs": [run.to_dict() for run in runs], "total_issues": sum(len(run.issues) for run in runs), } if ns.json: print(json.dumps(summary, indent=2, ensure_ascii=False)) else: verbose = not getattr(ns, "quiet", False) self._print_human_summary(summary, verbose=verbose) return not any_failed # ------------------------------------------------------------------ # Group dispatch # ------------------------------------------------------------------ def _run_group( self, group: str, scope: Optional[str], root: Path, ns: argparse.Namespace, ) -> List[ValidationRunResult]: """Run checks in *group*. Selection rules (in order): 1. If `scope` is set: run that one scope (regardless of its `default` flag). 2. Otherwise, run every scope in the group whose `default=True`. 3. If `--all-scopes` is set: also include `default=False` scopes. This three-way distinction is what lets pre-commit run a curated set without flag soup, while ad-hoc audits (`--all-scopes`) and targeted runs (`--scope`) still reach every scope on demand. """ all_scopes = getattr(ns, "all_scopes", False) results: List[ValidationRunResult] = [] for scope_obj in self.GROUPS[group]: scope_name = scope_obj.name method_name = scope_obj.runner if scope: if scope != scope_name: continue elif not all_scopes and not scope_obj.default: # Default group invocation skips scopes flagged default=False. # Run them via `--scope ` or `--all-scopes`. continue method = getattr(self, method_name) # Some runners need extra kwargs if method_name == "_run_refs": checks_code = ns.citations_in_code or (not ns.citations_in_code and not ns.citations_in_raw) checks_raw = ns.citations_in_raw or (not ns.citations_in_code and not ns.citations_in_raw) results.append(method(root, citations_in_code=checks_code, citations_in_raw=checks_raw)) elif method_name == "_run_inline_refs": results.append(method(root, check_patterns=ns.check_patterns, check_scope=getattr(ns, 'check_scope', False))) elif method_name == "_run_attr_latex_leaks": results.append(method(root, include_lightbox=getattr(ns, 'include_lightbox', False))) elif method_name == "_run_duplicate_labels": # Curated default for duplicates: figures + tables + listings only. # Section / equation dup-keys are legitimately ambiguous in vol2 # WIP (forward-referenced labels not yet authored) and would # block every commit. Explicit type flags from the user override # this default and switch to the user-selected subset. explicit = (ns.figures or ns.tables or ns.sections or ns.equations or ns.listings or ns.all_types) if explicit: label_types = self._selected_label_types(ns) else: label_types = { k: v for k, v in LABEL_DEF_PATTERNS.items() if k in ("Figure", "Table", "Listing") } results.append(method(root, label_types)) elif method_name == "_run_unreferenced_labels": # Curated default for orphans: all label types. The vol filter # comes from --vol1 / --vol2 at the path level (vol2 has many # forward references not yet authored — pre-commit pins to # --vol1 to avoid blocking commits on legitimate forwards). results.append(method(root, self._selected_label_types(ns))) elif method_name == "_run_check_references": results.append(method(root, ns)) elif method_name == "_run_sources": results.append(method(root, ns)) elif method_name == "_run_epubcheck": # Thresholds come from --max-fatal / --max-errors; when not # supplied on the CLI we keep max_errors=None (unlimited). # The ratchet takes over when --baseline is supplied. results.append(method( root, max_fatal=ns.max_fatal, max_errors=ns.max_errors, baseline_path=ns.baseline, update_baseline=ns.update_baseline, )) elif method_name == "_run_epub_hygiene": results.append(method(root, fix=getattr(ns, 'fix', False))) elif method_name == "_run_pdf_verify": results.append(method(root, vol1=ns.vol1, vol2=ns.vol2, log_path=getattr(ns, 'pdf_log', None))) elif method_name == "_run_pdf_numbering": results.append(method(root, vol1=ns.vol1, vol2=ns.vol2)) elif method_name == "_run_pdf_table_spacing": results.append(method(root, vol1=ns.vol1, vol2=ns.vol2)) else: results.append(method(root)) return results def _print_check_help(self) -> None: """Print a nicely formatted help for the check command.""" table = Table(show_header=True, header_style="bold cyan", box=None) table.add_column("Group", style="cyan", width=14) table.add_column("Scopes", style="yellow", width=38) table.add_column("Description", style="white", width=32) descriptions = { "refs": "References, citations, inline Python, self-ref", "cli": "Public Binder command contract (help, reset, removed aliases)", "labels": "Duplicate labels, orphans, fig-label underscores", "headers": "Section header IDs ({#sec-...}), H1-H5 case policy (MIT Press §10.3.1)", "bib": "Bibliography hygiene — schema + canonical forms (§5)", "footnotes": "Placement, integrity, cross-chapter duplicates, sentence-case first letter", "figures": "Captions, float flow, image files", "markup": "Low-level markup (patterns, div-fences, dropcaps)", "prose": "Prose style (contractions, dup words, ASCII, above/below, Acknowledgments)", "punctuation": "Em-dash, slash, vs. period, e.g./i.e. comma, en-dash ranges", "numbers": "Units + percent spacing, binary units, percent-in-captions, currency", "math": "\\times spacing, attr-leaks, fmt/suffix discipline (LEGO), multiplier prose, optional render audit", "structure": "Heading levels, H2 landings, parts, Purpose-unnumbered", "code": "Python code blocks (echo: false, _str/_math export hygiene)", "tables": "Grid tables → pipe, table content hygiene, caption-required", "listings": "Code listings carry lst-cap when labeled", "index": "Index placement (\\index{} outside headings/callouts)", "images": "Image file formats, external URLs", "json": "JSON file syntax validation", "units": "Physics engine unit conversion tests", "spelling": "Prose and TikZ spell checking (requires aspell)", "epub": "EPUB hygiene (source), epubcheck (built), structure (legacy)", "pdf": "Built PDF cross-ref / LaTeX undefined-ref / traceback scan", "registry": "Constants → registry migration gates (sources, tests, appendix)", "sources": "Source citation analysis and validation", "references": "Bibliography vs academic DBs (hallucinator)", "content": "Content tree (shared/, frontmatter/ required)", } for group_name, checks in self.GROUPS.items(): # Show curated scopes plain; mark default=False ones with a dim asterisk # so the user knows they exist but require --all-scopes / --scope opt-in. parts = [] for sc in checks: parts.append(sc.name if sc.default else f"{sc.name}*") table.add_row(group_name, ", ".join(parts), descriptions.get(group_name, "")) table.add_row("all", "(every group)", "Run every group's curated set; add --all-scopes for opt-in too") console.print(Panel(table, title="binder check [--scope ]", border_style="cyan")) console.print("[dim]Examples:[/dim]") console.print(" [cyan]./binder check refs[/cyan] [dim]# curated default scopes for refs[/dim]") console.print(" [cyan]./binder check refs --scope citations[/cyan] [dim]# one specific scope[/dim]") console.print(" [cyan]./binder check refs --all-scopes[/cyan] [dim]# every scope including opt-in (* in table)[/dim]") console.print(" [cyan]./binder check figures --vol1[/cyan] [dim]# default figure scopes, Vol I only[/dim]") console.print(" [cyan]./binder check all[/cyan] [dim]# every group, default scopes[/dim]") console.print(" [cyan]./binder check help[/cyan] [dim]# per-group error codes + guidance[/dim]") console.print("[dim]An asterisk (*) marks a scope that is default=False — runnable via --scope or --all-scopes.[/dim]") console.print() # ------------------------------------------------------------------ def _print_group_help(self, group: str) -> None: """Dispatch to per-group help. Falls back to a generic listing.""" if group == "epub": self._print_epub_help() return # Generic fallback: list scopes, default flag, runner method, and the # short note (if any). The Default column is the contract pre-commit / # CI rely on. scopes = self.GROUPS.get(group, []) table = Table(show_header=True, header_style="bold cyan", box=None) table.add_column("Scope", style="yellow") table.add_column("Default", style="green", width=8) table.add_column("Note", style="white") table.add_column("Runner", style="dim") for sc in scopes: table.add_row( sc.name, "yes" if sc.default else "opt-in", _rich_escape(sc.note) if sc.note else "", sc.runner, ) console.print(Panel(table, title=f"binder check {group}", border_style="cyan")) console.print(f"[dim]Run [cyan]./binder check {group}[/cyan] for the curated set " f"(Default = yes), [cyan]--scope [/cyan] for one specific scope, " f"or [cyan]--all-scopes[/cyan] to include opt-in scopes.[/dim]") console.print() def _print_epub_help(self) -> None: """Dedicated help for the `epub` group: the three scopes, the error codes each can surface, and how to fix them at source. This panel exists because the `epub` group is the most common source of unexpected failures for new contributors — the error codes (RSC-005, RSC-016, RSC-020, OPF-014, smoke-css-*) are cryptic in isolation, but map cleanly to a few source-level patterns. Printing the mapping here saves cross-referencing the README when a CI failure is in hand. """ # --- Scopes panel ----------------------------------------------- scope_table = Table(show_header=True, header_style="bold cyan", box=None) scope_table.add_column("Scope", style="yellow", width=12) scope_table.add_column("When to run", style="white", width=30) scope_table.add_column("Cost", style="dim", width=18) scope_table.add_column("Needs", style="dim", width=12) scope_table.add_row("hygiene", "Every commit (pre-commit)", "<1s, scans source", "—") scope_table.add_row("smoke", "After build, for reader compat", "~200ms, scans built EPUB", "—") scope_table.add_row("epubcheck", "CI, before release", "~7s per volume", "Java + epubcheck") console.print(Panel(scope_table, title="./binder check epub — scopes", border_style="cyan")) # --- Error code table ------------------------------------------ # Scope is derivable from the code prefix: svg-/bib-* = hygiene, # smoke-* = smoke, RSC-*/OPF-* = epubcheck. Dropping the column # leaves room for readable Trigger / Fix copy. code_table = Table(show_header=True, header_style="bold cyan", box=None) code_table.add_column("Code", style="red", width=32) code_table.add_column("Trigger", style="white", width=30) code_table.add_column("Fix at source", style="green", width=32) rows = [ ("svg-c0", "C0 control char in SVG aria-label", "Strip in Python plot title; or --fix"), ("svg-dupe-marker", "Duplicate in one SVG", "Delete duplicate; or --fix"), ("bib-url-escape-underscore", r"\_ in bib url= or http doi=", r"\_ → _ in the .bib; or --fix"), ("bib-url-escape-percent", r"\% in bib URL field", r"\% → % in the .bib; or --fix"), ("bib-url-raw-angle", "raw < or > in bib URL field", "%3C / %3E encode; or --fix"), ("smoke-css-custom-property-decl", "--var-name: value; in packaged CSS", "Inline the literal value"), ("smoke-css-custom-property-use", "var(--x) in packaged CSS", "Inline the literal value"), ("smoke-external-resource", "src=/, C0)", "epub_postprocess sanitizes; fix at source if it recurs"), ("RSC-005", "Malformed markup (alt on wrong elem)", "Fix the emitter in the Quarto filter"), ("RSC-020", "Invalid URL syntax in href", "Same as bib-url-* above"), ("RSC-012", "Broken fragment ID", "resolve_cross_references.py handles this"), ("OPF-014", "Missing mathml / other OPF property", "epub_postprocess aligns nav mathml property"), ] for code, trigger, fix in rows: code_table.add_row(code, trigger, fix) console.print(Panel(code_table, title="Error codes", border_style="red")) # --- Command quick reference ----------------------------------- console.print("[bold]Common invocations:[/bold]") console.print(" [cyan]./binder check epub[/cyan] [dim]# hygiene + smoke + epubcheck[/dim]") console.print(" [cyan]./binder check epub --scope hygiene[/cyan] [dim]# source-only, fast[/dim]") console.print(" [cyan]./binder check epub --scope hygiene --fix[/cyan] [dim]# auto-repair source issues[/dim]") console.print(" [cyan]./binder check epub --scope epubcheck --max-fatal 0 --max-errors 0[/cyan]") console.print(" [dim] # CI-style strict gating[/dim]") console.print(" [cyan]./binder check epub --json[/cyan] [dim]# structured output[/dim]") console.print(" [cyan]./binder build epub --vol1 --skip-hygiene[/cyan] [dim]# emergency build bypass[/dim]") console.print() console.print("[dim]Deep dive: book/cli/README.md → \"EPUB Checks — Two Layers, One CLI Surface\"[/dim]") console.print() # ------------------------------------------------------------------ def _resolve_path(self, path_arg: Optional[str], vol1: bool, vol2: bool) -> Path: if path_arg: path = Path(path_arg) if not path.is_absolute(): path = (Path.cwd() / path).resolve() return path base = self.config_manager.book_dir / "contents" if vol1 and not vol2: return base / "vol1" if vol2 and not vol1: return base / "vol2" return base def _selected_label_types(self, ns: argparse.Namespace) -> Dict[str, List[re.Pattern[str]]]: explicit = ns.figures or ns.tables or ns.sections or ns.equations or ns.listings if ns.all_types: return LABEL_DEF_PATTERNS if explicit: selected: Dict[str, List[re.Pattern[str]]] = {} if ns.figures: selected["Figure"] = LABEL_DEF_PATTERNS["Figure"] if ns.tables: selected["Table"] = LABEL_DEF_PATTERNS["Table"] if ns.sections: selected["Section"] = LABEL_DEF_PATTERNS["Section"] if ns.equations: selected["Equation"] = LABEL_DEF_PATTERNS["Equation"] if ns.listings: selected["Listing"] = LABEL_DEF_PATTERNS["Listing"] return selected # default: all label types return LABEL_DEF_PATTERNS def _qmd_files(self, root: Path) -> List[Path]: if root.is_file(): return [root] if root.suffix == ".qmd" else [] return sorted( p for p in root.rglob("*.qmd") if "_shelved" not in p.name ) def _bib_files(self, root: Path) -> List[Path]: if root.is_file(): return [root] if root.suffix == ".bib" else [] return sorted(root.rglob("*.bib")) def _read_text(self, path: Path) -> str: try: return path.read_text(encoding="utf-8") except UnicodeDecodeError: return path.read_text(encoding="utf-8", errors="ignore") def _relative_file(self, path: Path) -> str: try: return str(path.relative_to(self.config_manager.book_dir)) except ValueError: return str(path) def _run_cli_contract(self, root: Path) -> ValidationRunResult: """Run read-only command-surface contract checks for Binder itself.""" start = time.time() from cli.checks import cli_contract repo_root = Path(__file__).resolve().parents[3] violations = cli_contract.run_contract(repo_root) issues = [ ValidationIssue( file=violation.file, line=violation.line, code=violation.code, message=violation.message, severity="error", context=violation.context, suggestion=violation.suggestion, ) for violation in violations ] return ValidationRunResult( name="cli-contract", description="Public Binder CLI command contract", files_checked=len(cli_contract.CASES), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_binder_canonical(self, root: Path) -> ValidationRunResult: """Assert book-content pre-commit hooks dispatch through ./book/binder.""" start = time.time() from cli.checks import binder_canonical repo_root = Path(__file__).resolve().parents[3] violations = binder_canonical.run_canonical(repo_root) issues = [ ValidationIssue( file=violation.file, line=violation.line, code=violation.code, message=violation.message, severity="error", context=violation.context, suggestion=violation.suggestion, ) for violation in violations ] return ValidationRunResult( name="binder-canonical", description="Book-content hooks must route through Binder", files_checked=1, issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_python_syntax(self, root: Path) -> ValidationRunResult: """Compile every ```{python} code block to catch syntax errors.""" start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] block_start_re = re.compile(r"^```\{python\}") block_end_re = re.compile(r"^```\s*$") for file in files: content = self._read_text(file) lines = content.split("\n") rel = str(file.relative_to(root)) if file.is_relative_to(root) else str(file) in_block = False block_lines: List[str] = [] block_start_line = 0 for i, line in enumerate(lines, start=1): if block_start_re.match(line): in_block = True block_lines = [] block_start_line = i continue if in_block and block_end_re.match(line): in_block = False # Skip YAML-style #| directives before compiling source_lines = [ ln for ln in block_lines if not ln.strip().startswith("#|") ] source = "\n".join(source_lines) if not source.strip(): continue try: compile(source, f"{rel}:{block_start_line}", "exec") except SyntaxError as exc: err_line = block_start_line + (exc.lineno or 1) issues.append(ValidationIssue( file=rel, line=err_line, code="python_syntax", message=f"Python syntax error: {exc.msg}", severity="error", context=(exc.text or "").strip()[:120], )) continue if in_block: block_lines.append(line) return ValidationRunResult( name="python-syntax", description="Validate Python code block syntax (compile check)", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_inline_python(self, root: Path) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] regex_checks = [ ("missing_backtick", re.compile(r"(? ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] fenced_code_pattern = re.compile(r"```\{([^}]+)\}(.*?)```", re.DOTALL) raw_block_pattern = re.compile(r"```\{=(html|latex|tex)\}(.*?)```", re.DOTALL | re.IGNORECASE) problematic_classes = {"tikz", "latex", "tex"} for file in files: content = self._read_text(file) if citations_in_code: for match in fenced_code_pattern.finditer(content): attrs = match.group(1) code_content = match.group(2) class_match = re.search(r"\.([A-Za-z][A-Za-z0-9_-]*)", attrs) cls = class_match.group(1).lower() if class_match else "unknown" if cls not in problematic_classes: continue for cite_match in CITATION_BRACKET_PATTERN.finditer(code_content): offset = match.start() + len(f"```{{{attrs}}}") + cite_match.start() line_no = content[:offset].count("\n") + 1 line = content.splitlines()[line_no - 1] if line_no - 1 < len(content.splitlines()) else "" issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="citation_in_code", message=f"Citation in .{cls} code block will not be processed", severity="error", context=line.strip()[:160], )) if citations_in_raw: for match in raw_block_pattern.finditer(content): raw_type = match.group(1).lower() block = match.group(2) for cite_match in CITATION_BRACKET_PATTERN.finditer(block): offset = match.start() + cite_match.start() line_no = content[:offset].count("\n") + 1 line = content.splitlines()[line_no - 1] if line_no - 1 < len(content.splitlines()) else "" issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="citation_in_raw", message=f"Citation in raw {raw_type} block will not be processed", severity="error", context=line.strip()[:160], )) return ValidationRunResult( name="refs", description="Validate citation/reference placement in raw/code blocks", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) _CALLOUT_OPEN_RE = re.compile(r"^:{3,}\s*\{([^}]*)\}") _ATTR_CLASS_RE = re.compile(r"\.([A-Za-z0-9_-]+)") _ATTR_ID_RE = re.compile(r"#([A-Za-z0-9_-]+)") def _run_principle_refs(self, root: Path) -> ValidationRunResult: """Validate semantic principle IDs and references. This is a markup/semantic invariant, not a style-capitalization pass: it does not decide whether a principle name should be lowercase in prose. It only enforces the reference mechanics in book-prose §4. """ start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] manual_principle_re = re.compile(r"\b[Pp]rinciple\s+\d+(?:\.\d+)?\b") bad_principle_ref_re = re.compile(r"\b[Pp]rinciple\s+\\ref\{(?!pri-)([^}]+)\}") cite_principle_re = re.compile(r"@pri-[A-Za-z0-9_-]+") for file in files: lines = self._read_text(file).splitlines() in_code = False for idx, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("```"): in_code = not in_code continue if in_code: continue opener = self._CALLOUT_OPEN_RE.match(stripped) if opener: attrs = opener.group(1) classes = set(self._ATTR_CLASS_RE.findall(attrs)) if "callout-principle" in classes: ids = self._ATTR_ID_RE.findall(attrs) if not any(label.startswith("pri-") for label in ids): issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="principle_missing_pri_id", message="Principle callout must use a semantic #pri-* ID", severity="error", context=stripped[:160], ) ) if any(label.startswith("nte-") for label in ids): issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="principle_uses_note_id", message="Principle callout uses #nte-*; use #pri-* for principles", severity="error", context=stripped[:160], ) ) manual_match = manual_principle_re.search(line) if manual_match: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="manual_principle_number", message="Avoid manual principle numbering; use Principle \\ref{pri-*}", severity="error", context=line.strip()[:160], ) ) bad_ref_match = bad_principle_ref_re.search(line) if bad_ref_match: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="principle_ref_non_pri", message="Principle references must point to \\ref{pri-*}", severity="error", context=line.strip()[:160], ) ) cite_match = cite_principle_re.search(line) if cite_match: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="principle_citation_syntax", message="Use Principle \\ref{pri-*}, not @pri-* citation syntax", severity="error", context=line.strip()[:160], ) ) return ValidationRunResult( name="principles", description="Principle callout IDs and reference mechanics", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _bibliography_for_qmd(self, file: Path) -> Optional[Path]: """Resolve the shared book references.bib for a book .qmd.""" try: rel = file.relative_to(self.config_manager.book_dir) except ValueError: return None parts = rel.parts if not ({"vol1", "vol2", "frontmatter", "backmatter"} & set(parts)): return None bib_file = self.config_manager.book_dir / "contents" / "references.bib" return bib_file if bib_file.exists() else None def _run_citations(self, root: Path) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] bib_key_pattern = re.compile(r"@\w+\{([^,\s]+)") for file in files: bib_file = self._bibliography_for_qmd(file) if bib_file is None: continue content = self._read_text(file) bib_content = self._read_text(bib_file) bib_keys = set(bib_key_pattern.findall(bib_content)) # Strip YAML frontmatter (--- ... --- at file top) to avoid email false positives qmd_content_no_code = re.sub(r"^---\n.*?\n---\n", "", content, flags=re.DOTALL) # Strip HTML style/script blocks to avoid CSS @media false positives qmd_content_no_code = re.sub(r"]*>.*?", "", qmd_content_no_code, flags=re.DOTALL) qmd_content_no_code = re.sub(r"```.*?```", "", qmd_content_no_code, flags=re.DOTALL) qmd_content_no_code = re.sub(r"`[^`]+`", "", qmd_content_no_code) refs = set(CITATION_REF_PATTERN.findall(qmd_content_no_code)) refs = {r.rstrip(".,;:") for r in refs if not r.startswith(EXCLUDED_CITATION_PREFIXES)} refs = {r for r in refs if not re.match(r"^\d+\.\d+", r)} missing = sorted(refs - bib_keys) for key in missing: line_no = self._line_for_token(content, f"@{key}") issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="missing_citation", message=f"Citation key @{key} missing in bibliography", severity="error", context=f"@{key}", )) return ValidationRunResult( name="citations", description="Validate citation keys against bibliography files", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) _SCAFFOLD_CITATION_KIND_RE = re.compile( r"^:::+\s*\{[^}]*\.(callout-learning-objectives|callout-takeaways)\b" ) _SCAFFOLD_DIV_OPEN_RE = re.compile(r"^:::+\s*\{") _SCAFFOLD_DIV_CLOSE_RE = re.compile(r"^:::+\s*$") _SCAFFOLD_PURPOSE_RE = re.compile(r"^##\s+Purpose\b") _SCAFFOLD_H2_RE = re.compile(r"^##\s+") def _load_scaffold_citation_baseline(self) -> Set[Tuple[str, str, str]]: try: data = json.loads(SCAFFOLD_CITATION_BASELINE_PATH.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError): return set() return { (entry.get("file", ""), entry.get("kind", ""), entry.get("key", "")) for entry in data.get("allowed", []) } @staticmethod def _scaffold_citation_kind(raw_kind: str) -> str: if raw_kind == "callout-learning-objectives": return "learning-objectives" if raw_kind == "callout-takeaways": return "takeaways" return raw_kind @classmethod def _is_bibliography_citation_key(cls, key: str) -> bool: key = key.rstrip(".,;:)") return ( bool(key) and not key.startswith(EXCLUDED_CITATION_PREFIXES) and not key.lower().startswith(EXCLUDED_CITATION_PREFIXES) and not re.match(r"^\d+\.\d+", key) ) def _run_scaffold_citations(self, root: Path) -> ValidationRunResult: """Block new bibliography citations in pedagogical scaffolding. Purpose sections, learning objectives, and takeaways frame the chapter; they should not carry provenance. Put source support in the body prose, technical callout where the claim is taught, figure/table caption, or appendix prose instead. """ start = time.time() files = self._qmd_files(root) baseline = self._load_scaffold_citation_baseline() issues: List[ValidationIssue] = [] for file in files: rel = self._relative_file(file) lines = self._read_text(file).splitlines() in_fence = False in_purpose = False callout_kind: Optional[str] = None callout_depth = 0 for line_no, line in enumerate(lines, start=1): if self._BIB_FENCE_RE.match(line): in_fence = not in_fence continue if in_fence: continue if self._SCAFFOLD_PURPOSE_RE.match(line): in_purpose = True elif in_purpose and self._SCAFFOLD_H2_RE.match(line): in_purpose = False if callout_kind: if self._SCAFFOLD_DIV_OPEN_RE.match(line): callout_depth += 1 elif self._SCAFFOLD_DIV_CLOSE_RE.match(line): callout_depth -= 1 if callout_depth <= 0: callout_kind = None continue else: match = self._SCAFFOLD_CITATION_KIND_RE.match(line) if match: callout_kind = self._scaffold_citation_kind(match.group(1)) callout_depth = 1 kind = "purpose" if in_purpose else callout_kind if not kind: continue for key in CITATION_REF_PATTERN.findall(line): key = key.rstrip(".,;:)") if not self._is_bibliography_citation_key(key): continue if (rel, kind, key) in baseline: continue issues.append(ValidationIssue( file=rel, line=line_no, code="citation_in_scaffold", message=( f"Bibliography citation @{key} appears in {kind}. " "Purpose, learning objectives, and takeaways should not carry citations." ), severity="error", context=line.strip()[:220], suggestion=( f"Move @{key} to the body prose, a technical callout/caption, " "or appendix passage where the sourced claim is actually taught; " f"leave the {kind} text citation-free." ), )) return ValidationRunResult( name="scaffold-citations", description="No bibliography citations in Purpose, learning objectives, or takeaways", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_duplicate_citation_year(self, root: Path) -> ValidationRunResult: """Flag `[@citekey] (YEAR)` patterns — the citation already renders the year. Authors sometimes write `[@foo1964] (1964)` as belt-and-suspenders, but Pandoc renders `[@foo1964]` as "(Foo 1964)", so the literal `(1964)` is redundant and ships as "(Foo 1964) (1964)". Applies to `[@key]`, `[-@key]`, and multi-citation brackets like `[@a; @b]`. Resolution: move the citation to the end of the sentence and drop the parenthetical year, OR — when the literal year genuinely differs from the cited paper's year (e.g. system deployment year vs. publication year) — reword to avoid two adjacent parenthetical years. """ start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] # `[-?@...]` followed by optional whitespace and a bare `(YEAR)`. # Year range 1800–2099 keeps things loose without false-matching # measurements like `(1964 ms)` or `(100 µs)`. dup_year = re.compile(r"\[-?@[^\]]+\]\s*\((1[89]\d{2}|20\d{2})\)") for file in files: lines = self._read_text(file).splitlines() in_code = False for idx, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("```"): in_code = not in_code continue if in_code: continue # Strip inline code spans so `[@foo] (1964)` inside backticks # — documentation of the anti-pattern itself — does not fire. scrubbed = re.sub(r"`[^`]+`", "", line) for m in dup_year.finditer(scrubbed): context = scrubbed[max(0, m.start() - 10) : min(len(scrubbed), m.end() + 10)].strip() issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="duplicate_citation_year", message=( "Redundant (YEAR) after citation — Pandoc already renders " "the year inside the citation. Move citation to sentence end " "or reword to avoid two parenthetical years." ), severity="error", context=context, ) ) return ValidationRunResult( name="duplicate-citation-year", description='Flag "[@citekey] (YEAR)" — citation already carries the year', files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_duplicate_citation_key(self, root: Path) -> ValidationRunResult: """Flag same-key duplicates in cite groups: `[@k; @k]` or `[@k] [@k]`. Both patterns render through citeproc as a visible duplicate "(Author Year) (Author Year)" because the renderer treats each appearance of the same key as a separate citation event. Two shapes caught: 1. **Same key inside one bracket** — `[@key; @key]` or `[@a; @b; @a]`. Common cause: copy-paste while authoring a multi-cite. The fix is to drop the duplicate key: `[@key]` or `[@a; @b]`. 2. **Same key in stacked brackets** — `[@key][@key]` or `[@key] [@key]`. Common cause: two separate cite-additions in the same paragraph that landed adjacent to each other. The fix is to remove one of the brackets, OR — if both cites genuinely anchor different facts — separate them with prose so the reader sees each cite anchor a distinct claim. Distinct from `duplicate-year` (which flags `[@k] (YEAR)`) and from `manual-bracket` (which flags hand-typed author/year + cite). Skips fenced code blocks and inline backticks so anti-pattern documentation does not fire. """ start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] # "[@key; ...; @key]" — same key appears twice within one # multi-cite bracket group, possibly with other keys between. # Backreference (?P=k) requires the second occurrence to match # the first capture exactly. Spans up to 8 keys per group to # keep the regex bounded; longer multi-cites are rare. same_bracket_dup = re.compile( r"\[@(?P[A-Za-z][\w:.-]*)" r"(?:\s*;\s*@[\w:.-]+){0,8}" r"\s*;\s*@(?P=k)\b" ) # "[@key][@key]" or "[@key] [@key]" — same key in adjacent # bracket groups. Allow whitespace, comma, semicolon, or period # between the brackets (typical sentence punctuation that does # not change the citeproc-duplicate outcome). stacked_same_dup = re.compile( r"\[@(?P[A-Za-z][\w:.-]*)\][\s,;.]*\[@(?P=k)\b" ) for file in files: lines = self._read_text(file).splitlines() in_code = False for idx, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("```"): in_code = not in_code continue if in_code: continue # Strip inline code spans so anti-pattern documentation # in backticks does not fire on the example string. scrubbed = re.sub(r"`[^`]+`", "", line) for m in same_bracket_dup.finditer(scrubbed): context = scrubbed[max(0, m.start() - 10) : min(len(scrubbed), m.end() + 10)].strip() issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="duplicate_citation_key_in_bracket", message=( f"Same key @{m.group('k')} appears twice inside one cite group. " "Citeproc renders each occurrence as a separate (Author Year), " "duplicating the rendered citation. Drop the redundant copy." ), severity="error", context=context, ) ) for m in stacked_same_dup.finditer(scrubbed): context = scrubbed[max(0, m.start() - 10) : min(len(scrubbed), m.end() + 10)].strip() issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="duplicate_citation_key_stacked", message=( f"Same key @{m.group('k')} appears in two adjacent cite brackets. " "Citeproc renders each as '(Author Year)', producing a visible " "(Author Year) (Author Year) duplicate. Remove one bracket, or " "separate the two facts with prose so each cite anchors a distinct claim." ), severity="error", context=context, ) ) return ValidationRunResult( name="duplicate-citation-key", description='Flag "[@k; @k]" or "[@k] [@k]" — same-key citeproc dup', files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_manual_bracket_citation(self, root: Path) -> ValidationRunResult: """Flag hand-typed author/year attributions in prose. Two failure modes are caught here, both rooted in the same anti-pattern: an author + year written by hand instead of letting citeproc render the name from a `@key` / `[@key]` citation. 1. **Manual + bracket** (citeproc duplicate). Writing "Jeon et al. (ATC 2019) [@jeon2019analysis]" or "Cohen & Welling (2016) [@c]" duplicates the in-text citation — citeproc already prints "Jeon et al. (2019)" / "Cohen and Welling (2016)" from the key. Prefer `[@key]` / `[@a; @b]` alone, or narrative `@key`. 1b. **Narrative + duplicated year**. Writing "@williams2009roofline introduced the model at UC Berkeley in 2009" repeats the year that the narrative citation already renders. If the year is only there to identify the cited work, move or drop it. If the year is a separate timeline fact, reword so the prose does not read like duplicated attribution. 2. **Bare attribution** (no bib citation at all). Writing "Tri Dao et al. at Stanford (2022)" or "Coffman et al. (1971)" with no accompanying `[@key]` or narrative `@bibkey` on the line bypasses the bibliography entirely — the cited work is not in references.bib and the rendered text reads as plain prose rather than a verifiable citation. This pattern shows up most often inside footnote definitions where authors hand-type a provenance line. Fix with narrative `@key` (renders as "Dao et al. (2022)") or restructure to drop the attribution when no bib entry exists. Shapes we flag (prose, not code fences; inline backticks scrubbed first): Manual + bracket (citeproc duplicate): - `… et al. ( … YEAR … ) [@key]` — *et al.* + paren + bracket. - `Surname & Surname ( … ) [@key]` — two-author + bracket. Bare attribution (no [@key] or narrative @bibkey on the line): - `… et al. ( … YEAR … )` — classic *et al.* + paren-year. - `… et al.` with no paren-year on line — e.g. "Sambasivan et al. found that…" — author + year claim with nothing for the reader to look up. - `Surname and Surname ( … YEAR … )` — e.g. "Frankle and Carbin (MIT, 2019)". - `Surname, Surname, and Surname ( … YEAR … )` — e.g. "Hinton, Vinyals, and Dean (Google, 2015)". - `(Surname and Surname, YEAR)` — e.g. "(Ioffe and Szegedy, 2015)". These six bare patterns share a fix: replace the manual attribution with narrative `@key` (which renders as "Author et al. (YEAR)") or `[@key]` as a fact anchor. If no bib entry exists, restructure to drop the date and author rather than leaving an unverifiable claim. We do **not** flag acronyms in parentheses that are not *author* lines, e.g. "DARTS (Differentiable …) [@liu2019darts]": that has no *et al.* and no *Word & Word* before the paren. The bare-attribution branches that involve a paren require a 4-digit publication year (19xx/20xx) inside the parenthetical, so "(5--10)% improvement" or "(in production)" do not trigger it. Fenced code and inline backtick docs of anti-patterns are skipped. """ start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] # "et al. ( … ) [@" # The intermediate `[^.\n\[(]{0,80}` allows phrases like "at Stanford" # or "in 2014" between `et al.` and the year-paren without crossing # a sentence boundary, an opening bracket, or a nested paren. et_al_bracket = re.compile( r"et al\.[^.\n\[(]{0,80}\([^)\n]{0,500}\)\s*,?\s*\[@", re.IGNORECASE, ) # "Cohen & Welling (ReCOL 2016) [@" — two Titlecase tokens and & (surname style) two_surname_amp_bracket = re.compile( r"\b[A-Z][a-z][a-zA-Z'-]*\s+&\s+[A-Z][a-z][a-zA-Z'-]*\s*" r"\([^)\n]{0,500}\)\s*,?\s*\[@" ) # "Hennessy and Patterson [@Patterson2021]" — two surnames + and + bracket # cite, with no paren-year between. Distinct from `two_surname_and_paren` # (which requires a paren-year and is a *bare* attribution) and from # `two_surname_amp_bracket` (which uses & and requires a paren). Citeproc # renders [@key] as "(Author and Author YEAR)" — duplicates the prose. two_surname_and_bracket = re.compile( r"\b[A-Z][a-z][a-zA-Z'-]+\s+and\s+[A-Z][a-z][a-zA-Z'-]+\s*,?\s*\[@" ) # "(Sweeney, 2002) [@sweeney2002k]" — single-author parenthetical year # immediately before a bracket cite. Citeproc renders [@key] as # "(Author YEAR)" so the (Surname, YEAR) prose duplicates it. single_paren_author_year_bracket = re.compile( r"\(\s*[A-Z][a-z][a-zA-Z'-]+,\s*(?:19|20)\d{2}\s*\)\s*,?\s*\[@" ) # "**Optimal Brain Damage (LeCun, 1989)** [@cite]:" — footnote bold head # whose parenthetical already names (Author, YEAR) or (YEAR), followed # immediately by [@cite]. Per the book's footnote convention (Dartmouth # Conference, AlexNet, HIPAA, EU AI Act, A11 Bionic, Optimal Brain # Damage, etc.), the bold head IS the attribution — adding a bracket # cite right after produces "(YEAR) (Author YEAR)" in the rendered output. bold_head_year_bracket = re.compile( r"\*\*[^*\n]*\([^)\n]*\b(?:19|20)\d{2}\b[^)\n]*\)\*\*\s*\[@" ) # "[^fn-X]: **Term** [@cite]: ..." — footnote bold head followed # immediately by a [@cite], even when the head has no parenthetical # year. citeproc still inserts "(Author YEAR)" right where you put # [@cite], producing rendered output like "**Federated Learning** # (Li et al. 2020): A distributed training paradigm…" — the cite # reads as part of the term name. Place the cite inside the body # sentence at a specific factual claim instead. footnote_head_then_bracket = re.compile( r"^\[\^fn-[a-z0-9-]+\]:\s*\*\*[^*\n]+\*\*\s*\[@", re.M, ) # "text.[@cite]" — period BEFORE the cite. Convention is the # opposite: the cite is part of the sentence, so it goes before the # terminal punctuation: "text [@cite]." Pandoc renders the wrong # order with the period jammed against the citation visually. period_before_bracket = re.compile(r"\S\.\[@[A-Za-z]") # "word[@cite]" — no space before the cite. A parenthetical citation # always takes a leading space in body prose. Excludes footnote # markers and reference labels. no_space_before_bracket = re.compile(r"[a-zA-Z]\[@(?![Ss]ec-|[Ff]ig-|[Tt]bl-|[Ee]q-|[Ll]st-|exr-|exm-|thm-|cor-|cnj-|def-|prp-|rem-|prf-|alg-)[A-Za-z]") # "[@a, @b]" — comma-separated multi-cite. Pandoc's citation syntax # requires semicolons: "[@a; @b]". comma_multicite = re.compile(r"\[@[A-Za-z][\w:.-]+,\s*@[A-Za-z]") # "et al. ( … YEAR … )" — bare attribution. # Requires a 4-digit publication year inside the paren so we don't # flag unrelated parentheticals. et_al_bare = re.compile( r"et al\.[^.\n\[(]{0,100}" r"\(\s*[^)\n]{0,60}\b(?:19|20)\d{2}\b[^)\n]{0,60}\)", re.IGNORECASE, ) # "… et al." anywhere on a line, with no paren-year nearby. # Catches "Sambasivan et al. found that…" — the author makes an # attribution claim but provides no citation for the reader to verify. # The line-level no-cite suppression below ensures we only flag prose # that genuinely lacks a [@key] or narrative @bibkey. et_al_loose = re.compile(r"\bet\s+al\.", re.IGNORECASE) # "Surname and Surname (… YEAR …)" — two-author parenthetical year. # Catches "Frankle and Carbin (MIT, 2019)". Requires both surnames to # start with a capital letter and be followed by lowercase, so common # phrases like "this and that" / "Apple and Microsoft (2024 deal)" # have a lower hit rate. We also require a 4-digit year inside the # paren to avoid matching bare phrases like "Cohen and Welling". two_surname_and_paren = re.compile( r"\b[A-Z][a-z][a-zA-Z'-]+\s+and\s+[A-Z][a-z][a-zA-Z'-]+\s*" r"\(\s*[^)\n]{0,80}\b(?:19|20)\d{2}\b[^)\n]{0,80}\)" ) # "Surname, Surname, and Surname (… YEAR …)" — three-author. # Catches "Hinton, Vinyals, and Dean (Google, 2015)". Optional Oxford # comma (`,?`) before "and". Same paren-year requirement as above. three_surname_paren = re.compile( r"\b[A-Z][a-z][a-zA-Z'-]+,\s+[A-Z][a-z][a-zA-Z'-]+,?\s+and\s+" r"[A-Z][a-z][a-zA-Z'-]+\s*" r"\(\s*[^)\n]{0,80}\b(?:19|20)\d{2}\b[^)\n]{0,80}\)" ) # "(Surname and Surname, YEAR)" — fully parenthesized two-author cite. # Catches "(Ioffe and Szegedy, 2015)". Tighter than the two-surname # form above because both surnames AND the year live inside one # parenthesis with a comma separator — almost always an author cite. paren_two_authors = re.compile( r"\(\s*[A-Z][a-z][a-zA-Z'-]+\s+and\s+[A-Z][a-z][a-zA-Z'-]+,\s*" r"(?:19|20)\d{2}\s*\)" ) # Cite detectors used to suppress the bare branch when the line # already has a real citation. Excludes Quarto cross-ref prefixes # (@sec-, @fig-, etc.) so those are not mistaken for bibkeys. bracket_cite = re.compile(r"\[@[A-Za-z]") narrative_cite = re.compile( r"@(?![Ss]ec-|[Ff]ig-|[Tt]bl-|[Ee]q-|[Ll]st-|exr-|exm-|thm-|cor-|cnj-|" r"def-|prp-|rem-|prf-|alg-)[A-Za-z][\w:-]*" ) def issue_et_al( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Hand-written *et al.* (…) before a bracket cite—use only [@key] " "(or @key in narrative prose), not both. Citeproc already expands " "the key; drop the parenthetical name/venue you typed before [." ), severity="error", context=context, ) def issue_ampersand( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Hand-typed *Author & Author* (…) before a bracket cite—use only " "[@key] (or @key in narrative), not both. Citeproc already prints " "names from the key; drop the duplicate author line." ), severity="error", context=context, ) def issue_bare_attribution( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="bare_manual_attribution", message=( "Hand-typed *Author et al. (YEAR)* with no bibliography citation " "on this line. Use narrative @key (renders as 'Author et al. " "(YEAR)') so the work appears in references.bib, or restructure " "to drop the attribution when no bib entry exists." ), severity="error", context=context, ) def issue_etal_no_cite( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="bare_etal_no_cite", message=( "Used *Author et al.* in prose with no citation on the line. " "Either add narrative @key (renders as 'Author et al. (YEAR)') " "or [@key] as a fact anchor, or rewrite to drop the named " "attribution when no bib entry exists." ), severity="error", context=context, ) def issue_two_authors_no_cite( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="bare_two_author_attribution", message=( "Hand-typed *Surname and Surname (YEAR)* with no bibliography " "citation on this line. Use narrative @key (renders as " "'Surname and Surname (YEAR)') or [@key] as a fact anchor, " "not the manual paren-year form." ), severity="error", context=context, ) def issue_three_authors_no_cite( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="bare_three_author_attribution", message=( "Hand-typed *Surname, Surname, and Surname (YEAR)* with no " "bibliography citation on this line. Replace with narrative " "@key (citeproc renders the names from the bib entry) or " "[@key] as a fact anchor." ), severity="error", context=context, ) def issue_paren_authors_no_cite( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="bare_paren_author_attribution", message=( "Hand-typed *(Surname and Surname, YEAR)* parenthetical " "with no bibliography citation on this line. Replace with " "[@key] — citeproc renders it the same way and links to " "the bib entry." ), severity="error", context=context, ) def issue_two_surname_and_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Hand-typed *Surname and Surname* before a bracket cite — " "citeproc already prints both authors from [@key]. Use " "narrative @key (renders as 'Surname and Surname (YEAR)') " "or [@key] alone, not both." ), severity="error", context=context, ) def issue_single_paren_year_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Hand-typed *(Surname, YEAR)* before a bracket cite — " "citeproc renders [@key] as '(Author YEAR)' immediately " "after, duplicating the prose. Use [@key] alone." ), severity="error", context=context, ) def issue_bold_head_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Footnote bold head **Term (Author, YEAR)** is followed by " "a bracket cite [@key]; citeproc renders '(Author YEAR)' " "right after, duplicating the head's parenthetical. Per " "book convention (Dartmouth Conference, AlexNet, HIPAA, " "EU AI Act, etc.), the bold head IS the attribution — drop " "the cite, or place it later in the body attached to a " "specific factual claim." ), severity="error", context=context, ) def issue_footnote_head_then_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="manual_bracket_citation", message=( "Footnote bold head **Term** followed immediately by " "[@cite] — citeproc inserts '(Author YEAR)' right after " "the term name, making the cite read as part of the term " "rather than attribution for the body. Place the cite " "inside the body sentence at a specific factual claim " "(e.g., '...without centralizing the raw information " "[@key].')." ), severity="error", context=context, ) def issue_narrative_year_dup( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="narrative_citation_year", message=( "Narrative citation already renders the cited work's year. " "Do not repeat the same year later in the clause; move it " "or remove it unless it is a genuinely separate timeline fact." ), severity="error", context=context, ) def issue_period_before_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="cite_placement", message=( "Period BEFORE the citation. Convention is " "\"text [@cite].\" not \"text.[@cite]\" — the citation " "is part of the sentence and goes before terminal " "punctuation." ), severity="error", context=context, ) def issue_no_space_before_bracket( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="cite_placement", message=( "Citation glued to the preceding word. Always insert a " "space: \"word [@cite]\" not \"word[@cite]\"." ), severity="error", context=context, ) def issue_comma_multicite( f: Path, line_num: int, context: str ) -> ValidationIssue: return ValidationIssue( file=self._relative_file(f), line=line_num, code="cite_placement", message=( "Comma-separated multi-cite [@a, @b]. Pandoc's citation " "syntax requires semicolons: [@a; @b]." ), severity="error", context=context, ) for file in files: lines = self._read_text(file).splitlines() in_code = False for idx, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("```"): in_code = not in_code continue if in_code: continue scrubbed = re.sub(r"`[^`]+`", "", line) for m in et_al_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_et_al(file, idx, context)) for m in two_surname_amp_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_ampersand(file, idx, context)) for m in two_surname_and_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_two_surname_and_bracket(file, idx, context)) for m in single_paren_author_year_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_single_paren_year_bracket(file, idx, context)) for m in bold_head_year_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_bold_head_bracket(file, idx, context)) if footnote_head_then_bracket.match(scrubbed): context = scrubbed[: min(len(scrubbed), 80)] issues.append(issue_footnote_head_then_bracket(file, idx, context)) for m in period_before_bracket.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_period_before_bracket(file, idx, context)) for m in no_space_before_bracket.finditer(scrubbed): # Skip footnote markers like text[^fn-...] which look similar but # the regex requires "[@" so footnote markers (which use "[^") don't match. context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_no_space_before_bracket(file, idx, context)) for m in comma_multicite.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_comma_multicite(file, idx, context)) for m in narrative_cite.finditer(scrubbed): key = m.group(0)[1:] key_year_match = re.search(r"(?:18|19|20)\d{2}", key) if not key_year_match: continue key_year = key_year_match.group(0) tail = scrubbed[m.end() : m.end() + 120] year_dup = re.search( rf"\b(?:in|at|on|from|during|since)\s*\(?{key_year}\)?\b" rf"|(?:\(\s*{key_year}\s*\))", tail, ) if year_dup: context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 80) ].strip() issues.append(issue_narrative_year_dup(file, idx, context)) # Bare attribution: only flag when the line carries no real # citation. A bracket cite anywhere or a narrative @bibkey # (i.e., @ followed by a non-cross-ref word) suppresses this. line_has_cite = bool( bracket_cite.search(scrubbed) or narrative_cite.search(scrubbed) ) # Skip lines that look like code-comment annotations rather # than body prose. Two signals: a leading `#` followed by # ALL-CAPS content (e.g. "# VERIFIED DATA"), or any # `arXiv:NNNN.NNNNN` token on the line. These appear inside # `{python}` cells whose opening fence is mis-detected when # the file has unbalanced fences elsewhere. looks_like_code_comment = ( bool(re.match(r"^\s*#\s+[A-Z]{3,}", line)) or "arXiv:" in scrubbed ) if not line_has_cite and not looks_like_code_comment: for m in et_al_bare.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_bare_attribution(file, idx, context)) # If et_al_bare did not fire but et al. appears anywhere # on the line, the author claim still lacks a citation. if not et_al_bare.search(scrubbed): for m in et_al_loose.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 30) : min(len(scrubbed), m.end() + 40) ].strip() issues.append(issue_etal_no_cite(file, idx, context)) for m in two_surname_and_paren.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_two_authors_no_cite(file, idx, context)) for m in three_surname_paren.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 5) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_three_authors_no_cite(file, idx, context)) for m in paren_two_authors.finditer(scrubbed): context = scrubbed[ max(0, m.start() - 10) : min(len(scrubbed), m.end() + 24) ].strip() issues.append(issue_paren_authors_no_cite(file, idx, context)) return ValidationRunResult( name="manual-bracket", description=( "Flag hand-typed Author et al. (YEAR) attributions: " "manual+bracket duplicates and bare prose without [@key]" ), files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_duplicate_labels(self, root: Path, label_types: Dict[str, List[re.Pattern[str]]]) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] definitions: Dict[str, List[Tuple[Path, int, str]]] = {} for file in files: lines = self._read_text(file).splitlines() in_code = False for idx, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("```"): in_code = not in_code continue if in_code: continue for label_type, patterns in label_types.items(): for pattern in patterns: for match in pattern.finditer(line): label = match.group(1) definitions.setdefault(label, []).append((file, idx, label_type)) for label, locations in definitions.items(): if len(locations) <= 1: continue for file, line_no, label_type in locations: issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="duplicate_label", message=f"Duplicate {label_type.lower()} label: {label}", severity="error", context=label, )) return ValidationRunResult( name="duplicate-labels", description="Detect duplicate label definitions", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_unreferenced_labels(self, root: Path, label_types: Dict[str, List[re.Pattern[str]]]) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] defined: Dict[str, Tuple[Path, int, str]] = {} references: Dict[str, List[Tuple[Path, int]]] = {} for file in files: lines = self._read_text(file).splitlines() in_code_block = False in_html_comment = False for idx, line in enumerate(lines, 1): # Skip three forms of non-rendered content: # 1. Fenced code blocks (```...```). `@xref` mentions in # Python cell box-comment headers (`# │ Context: @sec-foo`) # are documentation that never renders. Counting them as # references produces false-positive unresolved-reference # errors for refs that target intentional documentation. # 2. HTML comments (``). Commented-out tikz # examples often contain ``` fence-looking lines, which # desynchronize the naive in_code_block toggle if not # tracked separately. Track comment state explicitly. # 3. Labels themselves are never defined inside fences or # HTML comments, so skipping the whole region is safe. stripped = line.lstrip() # Update HTML-comment state first — comment markers may sit # on the same line as content (rare) or span many lines. if not in_code_block: if "" not in line[line.index("" in line: in_html_comment = False continue # Pseudocode algorithm labels are declared as a chunk option # (`#| label: algo-xyz`) INSIDE a ```pseudocode fence, so the # code-block skip below would hide them and leave every # @algo-/@Algo- reference unresolved. A chunk-option label is a # genuine definition wherever it lives, so harvest it before the # skip. (mlsysbook-ext/pseudocode; native @alg- collides with # Quarto's algorithm theorem env, hence the algo- prefix.) algo_def = PSEUDOCODE_LABEL_PATTERN.match(stripped) if algo_def: defined.setdefault(algo_def.group(1), (file, idx, "Algorithm")) if stripped.startswith("```"): in_code_block = not in_code_block continue if in_code_block: continue for label_type, patterns in label_types.items(): for pattern in patterns: for match in pattern.finditer(line): defined.setdefault(match.group(1), (file, idx, label_type)) for match in LABEL_REF_PATTERN.finditer(line): label = match.group(1) # Normalize capitalized Quarto prefix (@Fig- -> fig-) label = label[0].lower() + label[1:] references.setdefault(label, []).append((file, idx)) # unreferenced definitions (skip section defaults, consistent with legacy behavior) for label, (file, line_no, label_type) in defined.items(): if label_type == "Section": continue if label not in references: issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="unreferenced_label", message=f"{label_type} label {label} is never referenced", severity="warning", context=label, )) # unresolved references defined_labels = set(defined.keys()) for label, locations in references.items(): if label in defined_labels: continue for file, line_no in locations: issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="unresolved_reference", message=f"Reference @{label} has no matching label definition", severity="error", context=f"@{label}", )) return ValidationRunResult( name="unreferenced-labels", description="Detect unreferenced labels and unresolved references", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) def _run_inline_refs(self, root: Path, check_patterns: bool, check_scope: bool = False) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] yaml_option_inline = re.compile(r"^#\|\s*(fig-cap|tbl-cap|lst-cap|fig-alt):\s*.*`\{python\}") inline_fstring = re.compile(r"`\{python\}\s*f\"[^`]+`") inline_func_call = re.compile(r"`\{python\}\s*\w+\([^`]+\)`") for file in files: lines = self._read_text(file).splitlines() refs: List[Tuple[int, str]] = [] compute_vars: Set[str] = set() compute_classes: Set[str] = set() in_cell = False for idx, line in enumerate(lines, 1): if CELL_START_PATTERN.match(line.strip()): in_cell = True continue if in_cell and CELL_END_PATTERN.match(line.strip()): in_cell = False continue if in_cell: cls_match = CLASS_DEF_PATTERN.match(line.strip()) if cls_match: compute_classes.add(cls_match.group(1)) assign = ASSIGN_PATTERN.match(line.strip()) if assign: compute_vars.add(assign.group(1)) tuple_assign = TUPLE_ASSIGN_PATTERN.match(line.strip()) if tuple_assign: for name in re.split(r'\s*,\s*', tuple_assign.group(1)): compute_vars.add(name.strip()) for match in INLINE_REF_PATTERN.finditer(line): refs.append((idx, match.group(1))) for line_no, ref in refs: if "." in ref: cls_name = ref.split(".", 1)[0] resolved = cls_name in compute_classes or cls_name in compute_vars else: resolved = ref in compute_vars if not resolved: issues.append(ValidationIssue( file=self._relative_file(file), line=line_no, code="undefined_inline_ref", message=f"Inline reference `{ref}` is not defined in python cells", severity="error", context=f"`{{python}} {ref}`", )) if check_patterns: in_grid = False for idx, line in enumerate(lines, 1): stripped = line.strip() if GRID_TABLE_SEP_PATTERN.match(stripped): in_grid = True elif in_grid and stripped and not stripped.startswith("|"): in_grid = False if in_grid and "`{python}" in line: issues.append(ValidationIssue( file=self._relative_file(file), line=idx, code="grid_table_inline_python", message="Inline Python in grid tables is unsupported", severity="error", context=stripped[:160], )) if inline_fstring.search(line): issues.append(ValidationIssue( file=self._relative_file(file), line=idx, code="inline_fstring", message="Inline f-string should be precomputed in Python cell", severity="warning", context=stripped[:160], )) if inline_func_call.search(line): issues.append(ValidationIssue( file=self._relative_file(file), line=idx, code="inline_function_call", message="Inline function call should be precomputed in Python cell", severity="warning", context=stripped[:160], )) if yaml_option_inline.search(line): issues.append(ValidationIssue( file=self._relative_file(file), line=idx, code="yaml_option_inline_python", message="Inline Python in YAML fig/tbl/lst metadata will not render", severity="error", context=stripped[:160], )) if check_scope: from book.tools.scripts.maintenance.validate_inline_refs import check_scope as _check_scope, BOOK_ROOT try: scope_warnings = _check_scope(file, verbose=False) for filepath, lineno, check_type, msg in scope_warnings: issues.append(ValidationIssue( file=self._relative_file(file), line=lineno, code=check_type.lower(), message=msg, severity="warning", context="", )) except Exception: pass return ValidationRunResult( name="inline-refs", description="Validate inline Python refs and rendering hazard patterns", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) # ------------------------------------------------------------------ # Headers (ported from manage_section_ids.py --verify) # ------------------------------------------------------------------ def _run_headers(self, root: Path) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] header_pat = re.compile(r"^(#{1,6})\s+(.+?)(?:\s*\{[^}]*\})?$") div_start_pat = re.compile(r"^:::\s*\{\.") div_end_pat = re.compile(r"^:::\s*$") code_block_pat = re.compile(r"^```[^`]*$") sec_id_pat = re.compile(r"\{#sec-[^}]+\}") for file in files: lines = self._read_text(file).splitlines() in_code = False in_div = False in_html_comment = False for idx, line in enumerate(lines, 1): stripped = line.strip() # HTML comments are tracked OUTSIDE code blocks. A code fence # inside an HTML comment (e.g. a hidden TikZ example) is just # text, not a real fence — toggling in_code on it would mis- # classify the rest of the file. See `_maintain_section_ids` # in maintenance.py for the matching writer-side fix. if not in_code: if "" not in stripped: in_html_comment = True continue if in_html_comment: if "-->" in stripped: in_html_comment = False continue if code_block_pat.match(stripped): in_code = not in_code continue if in_code: continue if div_start_pat.match(stripped): in_div = True continue if div_end_pat.match(stripped): in_div = False continue if in_div: continue match = header_pat.match(line) if not match: continue # Extract existing attributes existing_attrs = "" if "{" in line: attrs_start = line.find("{") attrs_end = line.rfind("}") if attrs_end > attrs_start: existing_attrs = line[attrs_start : attrs_end + 1] if ".unnumbered" in existing_attrs: continue if not sec_id_pat.search(line): title = match.group(2).strip() issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="missing_section_id", message=f"Header missing section ID: {title}", severity="error", context=line.strip()[:160], ) ) return ValidationRunResult( name="headers", description="Verify section headers have {#sec-...} IDs", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) # ------------------------------------------------------------------ # Footnote Placement (ported from check_forbidden_footnotes.py) # ------------------------------------------------------------------ def _run_footnote_placement(self, root: Path) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] fn_pat = re.compile(r"\[\^fn-[\w-]+\]") fn_def_pat = re.compile(r"^\[\^(fn-[\w-]+)\]:") inline_fn_pat = re.compile(r"\^\[[^\]]+\]") list_item_pat = re.compile(r"^(?P\s*)(?P(?:[-+*]|\d+[.)]))\s+") table_sep_pat = re.compile(r"^\|[\s\-:+]+\|") # Citation-then-footnote: visually anchors the footnote to the # bibliographic reference instead of the concept term. \s* # tolerates any whitespace between the two tokens (zero, single # space, multiple spaces) — the misanchoring is the same # regardless of how they are spaced. # See book-prose.md §5 ("Footnote Marker Placement"). cite_then_fn_pat = re.compile(r"(\[@[^\]\s]+\])\s*(\[\^fn-[\w-]+\])") for file in files: lines = self._read_text(file).splitlines() div_depth = 0 div_start_line = 0 def previous_list_item(line_index: int) -> Optional[Tuple[int, int]]: """Return (line, indent) for the list item immediately before a block.""" for prev_index in range(line_index - 1, -1, -1): prev_line = lines[prev_index] prev_stripped = prev_line.strip() if not prev_stripped: continue if fn_def_pat.match(prev_stripped): return None if ( prev_stripped.startswith("#") or prev_stripped.startswith(":::") or prev_stripped.startswith("```") or prev_stripped.startswith("|") ): return None marker = list_item_pat.match(prev_line) if marker: return prev_index + 1, len(marker.group("indent")) if not prev_line.startswith((" ", "\t")): return None return None def next_list_item_after_footnote(line_index: int) -> Optional[Tuple[int, int]]: """Return (line, indent) when a later list item follows before prose.""" for next_index in range(line_index + 1, len(lines)): next_line = lines[next_index] next_stripped = next_line.strip() if not next_stripped: continue if fn_def_pat.match(next_stripped): continue marker = list_item_pat.match(next_line) if marker: return next_index + 1, len(marker.group("indent")) if next_line.startswith((" ", "\t")): continue return None return None for idx, line in enumerate(lines, 1): stripped = line.strip() # Track div nesting if re.match(r"^:{3,4}\s*\{", stripped) or re.match(r"^:{3,4}\s+\w", stripped): div_depth += 1 if div_depth == 1: div_start_line = idx elif re.match(r"^:{3,4}\s*$", stripped): if div_depth > 0: div_depth -= 1 if div_depth == 0: div_start_line = 0 fn_def = fn_def_pat.match(stripped) if fn_def: prev_list = previous_list_item(idx - 1) next_list = next_list_item_after_footnote(idx - 1) if prev_list and next_list: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_def_interrupts_list", message=( f"Footnote definition [^{fn_def.group(1)}]: appears between " f"list items (previous item line {prev_list[0]}, next item " f"line {next_list[0]}). Move the definition after the complete " f"Markdown list so Pandoc preserves the list structure." ), severity="error", context=stripped[:80], ) ) # Check inline footnotes (always forbidden) for m in inline_fn_pat.finditer(line): issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="inline_footnote", message=f"Inline footnote syntax; use [^fn-name] reference format", severity="error", context=m.group(0)[:80], ) ) # Citation immediately followed by footnote marker # ([@cite][^fn-...]) — the marker should anchor to the # concept term, not the bibliographic reference. # rich-escape the bracketed tokens so [@key] doesn't get # eaten by Rich's variable-interpolation markup. for m in cite_then_fn_pat.finditer(line): cite, fn = m.group(1), m.group(2) issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_after_citation", message=( f"Footnote {_rich_escape(fn)} placed immediately " f"after citation {_rich_escape(cite)}; the marker " f"should anchor to the concept term it " f"elaborates, not the bibliographic reference. " f"Move the marker leftward onto the term (see " f"book-prose.md §5)." ), severity="error", context=_rich_escape(m.group(0)[:80]), ) ) footnotes = fn_pat.findall(line) if not footnotes: continue # Table cell check if stripped.startswith("|") and stripped.count("|") >= 2 and not table_sep_pat.match(stripped): for fn in footnotes: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_in_table", message=f"Footnote {fn} in table cell", severity="error", context=stripped[:80], ) ) # YAML caption check if re.match(r"^\s*(fig-cap|tbl-cap):", line): cap_type = "figure" if "fig-cap" in line else "table" for fn in footnotes: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code=f"footnote_in_{cap_type}_caption", message=f"Footnote {fn} in {cap_type} caption", severity="error", context=stripped[:80], ) ) # Markdown caption check if re.match(r"^:\s*\*\*[^*]+\*\*:", line): for fn in footnotes: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_in_markdown_caption", message=f"Footnote {fn} in markdown caption", severity="error", context=stripped[:80], ) ) # Callout title check if re.match(r"^:{3,4}\s*\{.*title=", stripped): title_match = re.search(r'title="([^"]*)"', line) if title_match and fn_pat.search(title_match.group(1)): for fn in fn_pat.findall(title_match.group(1)): issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_in_callout_title", message=f"Footnote {fn} in callout title (breaks LaTeX)", severity="error", context=stripped[:80], ) ) # Div block check if div_depth > 0 and div_start_line != idx: for fn in footnotes: issues.append( ValidationIssue( file=self._relative_file(file), line=idx, code="footnote_in_div", message=f"Footnote {fn} inside div block (started line {div_start_line})", severity="error", context=stripped[:80], ) ) return ValidationRunResult( name="footnote-placement", description="Check footnotes in forbidden locations and citation-adjacent placements", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) # ------------------------------------------------------------------ # Footnote Refs (ported from footnote_cleanup.py --validate) # ------------------------------------------------------------------ def _run_footnote_refs(self, root: Path) -> ValidationRunResult: start = time.time() files = self._qmd_files(root) issues: List[ValidationIssue] = [] ref_pat = re.compile(r"\[\^([^]]+)\]") def_pat = re.compile(r"^\[\^([^]]+)\]:\s*(.+)$", re.MULTILINE) for file in files: content = self._read_text(file) lines = content.split("\n") # Collect definitions fn_defs: Dict[str, str] = {} for m in def_pat.finditer(content): fn_defs[m.group(1)] = m.group(2) # Collect references (excluding definition lines themselves) fn_refs: Dict[str, List[int]] = defaultdict(list) for line_num, line in enumerate(lines, 1): for m in ref_pat.finditer(line): fn_id = m.group(1) dm = def_pat.match(line) if dm and dm.group(1) == fn_id: continue # definition line, not a reference fn_refs[fn_id].append(line_num) # Undefined references for fn_id in sorted(set(fn_refs.keys()) - set(fn_defs.keys())): first_line = fn_refs[fn_id][0] issues.append( ValidationIssue( file=self._relative_file(file), line=first_line, code="undefined_footnote_ref", message=f"Undefined footnote reference: [^{fn_id}]", severity="error", context=f"[^{fn_id}]", ) ) # Unused definitions for fn_id in sorted(set(fn_defs.keys()) - set(fn_refs.keys())): def_line = self._line_for_token(content, f"[^{fn_id}]:") issues.append( ValidationIssue( file=self._relative_file(file), line=def_line, code="unused_footnote_def", message=f"Unused footnote definition: [^{fn_id}]", severity="warning", context=f"[^{fn_id}]:", ) ) # Duplicate definitions def_counts: Dict[str, int] = defaultdict(int) for line in lines: dm = re.match(r"^\[\^([^]]+)\]:", line) if dm: def_counts[dm.group(1)] += 1 for fn_id, count in def_counts.items(): if count > 1: issues.append( ValidationIssue( file=self._relative_file(file), line=self._line_for_token(content, f"[^{fn_id}]:"), code="duplicate_footnote_def", message=f"Duplicate footnote definition ({count}x): [^{fn_id}]", severity="error", context=f"[^{fn_id}]:", ) ) # Missing blank line before footnote definition # Pandoc requires footnote definitions to start a new block. # Without a preceding blank line, Pandoc treats the definition # as continuation text and renders [^fn-name] as literal text. fn_def_line_pat = re.compile(r"^\[\^[^\]]+\]:") for idx, line in enumerate(lines): if fn_def_line_pat.match(line) and idx > 0: prev = lines[idx - 1] if prev.strip(): # previous line is not blank fn_match = re.match(r"^\[\^([^\]]+)\]:", line) fn_id_str = fn_match.group(1) if fn_match else "?" issues.append( ValidationIssue( file=self._relative_file(file), line=idx + 1, code="footnote_missing_blank_line", message=( f"Footnote definition [^{fn_id_str}] has no blank line before it — " f"Pandoc will not parse it as a footnote" ), severity="error", context=f"prev: {prev.strip()[:60]}", ) ) return ValidationRunResult( name="footnote-refs", description="Validate footnote references and definitions", files_checked=len(files), issues=issues, elapsed_ms=int((time.time() - start) * 1000), ) # ------------------------------------------------------------------ # Figures (ported from check_figure_completeness.py) # ------------------------------------------------------------------ _CAPTION_HEAD_ATTR_RE = re.compile( r"""\b(?Pfig-cap|tbl-cap|lst-cap)\s*=\s*"(?P[^"]*)\"""" ) _MARKDOWN_CAPTION_HEAD_RE = re.compile( r"""^\s*:\s+(?P.+?)\s+\{#(?P