Files

398 lines
15 KiB
Python
Executable File

#!/usr/bin/env python3
"""
PDF math-rendering audit for MLSysBook chapters.
For each chapter:
1. Builds a PDF via `./binder/binder build pdf <vol>/<chap>`.
A build failure is itself a strong signal of a math-rendering bug
(raw \\command outside math mode causes LaTeX to error out).
2. Extracts text via `pdftotext` and applies the same leak-pattern
detector used by the HTML audit. Catches LaTeX leakage that compiled
successfully but typeset literally.
3. Renders pages to PNG via `pdftoppm -r 150` so a human can spot-check.
Optionally limited to the first N pages for sampling.
NOTE: This script must NOT run concurrently with `audit_math_rendering.py`
or any other `binder build` invocation -- they all mutate the shared
`books/_quarto.yml`.
Usage (run from repo root):
python3 binder/tools/audit/math_render/audit_math_pdf.py vol1/introduction vol2/inference
python3 binder/tools/audit/math_render/audit_math_pdf.py --fixed # only chapters we touched
python3 binder/tools/audit/math_render/audit_math_pdf.py --all # every chapter
python3 binder/tools/audit/math_render/audit_math_pdf.py --max-pages 0 # render every page
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import subprocess
import sys
import time
from dataclasses import dataclass, field, asdict
from pathlib import Path
REPO = Path(__file__).resolve().parents[4]
BINDER = REPO / "binder" / "binder"
BUILD_DIRS = {
"vol1": REPO / "books" / "_build" / "pdf-vol1",
# binder writes vol2 PDFs into the per-volume directory if it's
# configured (config/_quarto-pdf-vol2.yml). If not present at runtime,
# the fallback in find_pdf() searches both candidate dirs.
"vol2": REPO / "books" / "_build" / "pdf-vol2",
}
OUT_DIR = REPO / "audit-pdf-output"
# Same leak patterns as HTML audit (kept in sync intentionally).
LEAK_PATTERNS: list[tuple[str, re.Pattern]] = [
("backslash-cmd", re.compile(r"\\(?:times|frac|approx|alpha|beta|gamma|delta|sigma|mu|eta|lambda|theta|sqrt|sum|int|cdot|leq|geq|neq|partial|nabla|infty|log|ln|exp|min|max|pm|mp|forall|exists|in|notin|to|gets|mapsto|rightarrow|Rightarrow|prod|equiv|sim|propto|land|lor|neg|text|mathbf|mathit|mathcal|mathbb)\b")), # codespell:ignore notin
("caret-brace", re.compile(r"(?<![A-Za-z0-9_])\^\{[^}\n]{1,40}\}")),
("under-brace", re.compile(r"(?<![A-Za-z0-9_])_\{[^}\n]{1,40}\}")),
("bare-pow10", re.compile(r"\b10\^\{[+-]?\d{1,4}\}")),
("dollar-math-text", re.compile(r"\$\\[a-zA-Z]+|\\[a-zA-Z]+\$")),
]
# Chapters that received our LaTeX rendering fixes (touched in commits
# 08bf690d0, 3ba9e8b06, adcb0b5e1).
FIXED_CHAPTERS = [
("introduction", "vol1"),
("benchmarking", "vol1"),
("appendix_dam", "vol1"),
("appendix_algorithm", "vol1"),
("model_compression", "vol1"),
("conclusion", "vol2"),
("inference", "vol2"),
("introduction", "vol2"),
("appendix_dam", "vol2"),
("appendix_c3", "vol2"),
("robust_ai", "vol2"),
]
@dataclass
class Leak:
"""One leak-pattern hit in extracted PDF text, with about 40 characters of context per side."""
pattern: str
match: str
context: str
@dataclass
class PdfReport:
"""Per-chapter result of the PDF math audit, serialized into the JSON report.
Paths are repo-relative. ``build_ok`` is also cleared when no PDF can be
found, and ``error`` then holds the reason or the tail of the build output.
"""
name: str
volume: str
qmd_path: str
build_ok: bool
build_seconds: float
pdf_path: str | None = None
pages: int = 0
image_dir: str | None = None
image_count: int = 0
leaks: list[Leak] = field(default_factory=list)
error: str | None = None
def list_all_chapters() -> list[tuple[str, str, Path]]:
"""Return ``(stem, volume, qmd_path)`` for every ``books/<vol>/*/*.qmd`` in vol1 and vol2.
Files whose names start with ``_`` are skipped, except the two notation partials.
"""
chapters = []
for vol in ("vol1", "vol2"):
contents = REPO / "books" / vol
for qmd in sorted(contents.glob("*/*.qmd")):
if qmd.name.startswith("_"):
continue
chapters.append((qmd.stem, vol, qmd))
return chapters
def build_pdf(name: str, volume: str) -> tuple[bool, float, str]:
"""Build one chapter PDF through the binder CLI and return ``(ok, seconds, output)``.
Side effect: replaces ``books/index.qmd`` (unlinking it first if it is a
symlink) with a copy of ``books/index-<volume>.qmd``; copy errors are
ignored. Runs ``python3 binder/binder build pdf <name> --<volume>`` with a
900-second timeout. ``ok`` requires a zero exit code and a completion phrase
in the combined output; a timeout returns False instead of raising.
"""
# Per-volume index: PDF builds need `index.qmd` to hold the content of
# `index-vol1.qmd` or `index-vol2.qmd`. CI copies it per job; we do the
# same here.
index_path = REPO / "books" / "index.qmd"
try:
if index_path.is_symlink():
index_path.unlink()
shutil.copyfile(REPO / "books" / f"index-{volume}.qmd", index_path)
except OSError:
pass
vol_flag = f"--{volume}"
t0 = time.time()
try:
proc = subprocess.run(
["python3", str(BINDER), "build", "pdf", name, vol_flag],
cwd=REPO,
capture_output=True,
text=True,
timeout=900,
)
except subprocess.TimeoutExpired:
return False, time.time() - t0, "build timed out after 900s"
secs = time.time() - t0
out = (proc.stdout or "") + "\n" + (proc.stderr or "")
ok = proc.returncode == 0 and (
"build completed" in out.lower()
or "output created" in out.lower()
or "render complete" in out.lower()
)
return ok, secs, out
def find_pdf(name: str, volume: str, qmd_path: Path) -> Path | None:
"""Locate the built PDF for a chapter, or return None.
Searches the volume's build directory, then both per-volume PDF directories.
In the first existing directory that contains any PDF it returns the
chapter-named or book-named PDF, else the most recently modified ``*.pdf``
there, which may be stale. ``qmd_path`` is unused.
"""
candidate_dirs = [BUILD_DIRS[volume],
REPO / "books" / "_build" / "pdf-vol1",
REPO / "books" / "_build" / "pdf-vol2"]
for base in candidate_dirs:
if not base.exists():
continue
candidates = [
base / f"{name}.pdf",
base / f"Machine-Learning-Systems-{volume.title()}.pdf",
base / "Machine-Learning-Systems.pdf",
base / "MLSysBook.pdf",
]
for c in candidates:
if c.exists():
return c
pdfs = sorted(base.glob("*.pdf"), key=lambda p: p.stat().st_mtime, reverse=True)
if pdfs:
return pdfs[0]
return None
def pdf_pages(pdf: Path) -> int:
"""Return the page count reported by ``pdfinfo``, or 0 if it fails or reports none."""
try:
out = subprocess.run(
["pdfinfo", str(pdf)], capture_output=True, text=True, timeout=30
).stdout
for line in out.splitlines():
if line.startswith("Pages:"):
return int(line.split(":")[1].strip())
except Exception:
pass
return 0
def extract_text(pdf: Path) -> str:
"""Return the layout-preserving text of ``pdf`` from ``pdftotext``, or "" on failure."""
try:
return subprocess.run(
["pdftotext", "-layout", str(pdf), "-"],
capture_output=True, text=True, timeout=120,
).stdout
except Exception:
return ""
def scan_text(text: str) -> list[Leak]:
"""Return the ``LEAK_PATTERNS`` hits in ``text``, stopping once more than 200 are found."""
leaks: list[Leak] = []
for label, pat in LEAK_PATTERNS:
for m in pat.finditer(text):
start, end = max(0, m.start() - 40), min(len(text), m.end() + 40)
ctx = text[start:end].replace("\n", " ").strip()
leaks.append(Leak(label, m.group(0), ctx))
if len(leaks) > 200:
return leaks
return leaks
def render_images(pdf: Path, dest: Path, max_pages: int) -> int:
"""Render pages to 150-dpi ``page-*.png`` files under ``dest`` and return how many exist.
``max_pages`` of 0 renders every page; otherwise only the first
``max_pages``. Prints the error and returns 0 if ``pdftoppm`` fails.
"""
dest.mkdir(parents=True, exist_ok=True)
args = ["pdftoppm", "-r", "150", "-png", str(pdf), str(dest / "page")]
if max_pages > 0:
args.extend(["-l", str(max_pages)])
try:
subprocess.run(args, check=True, capture_output=True, timeout=600)
except Exception as e:
print(f" pdftoppm failed: {e}")
return 0
return len(list(dest.glob("page-*.png")))
def audit_chapter(name: str, volume: str, qmd_path: Path,
max_pages: int, build: bool) -> PdfReport:
"""Optionally build, then locate and scan one chapter's PDF.
The PDF is copied to ``audit-pdf-output/<volume>/<name>/<name>.pdf`` (the
chapter directory is deleted and recreated first) so the next build cannot
overwrite it; page count, leaks, and PNG renders are then recorded. Returns
early with ``error`` set when the build fails or no PDF is found.
"""
rep = PdfReport(
name=name, volume=volume,
qmd_path=str(qmd_path.relative_to(REPO)),
build_ok=True, build_seconds=0.0,
)
if build:
ok, secs, output = build_pdf(name, volume)
rep.build_ok = ok
rep.build_seconds = secs
if not ok:
rep.error = output[-2000:]
return rep
pdf = find_pdf(name, volume, qmd_path)
if pdf is None:
rep.error = "pdf output not found"
rep.build_ok = False
return rep
# Persist the PDF to a chapter-specific path; the next build will
# otherwise overwrite the per-volume output.
chapter_dir = OUT_DIR / volume / name
if chapter_dir.exists():
shutil.rmtree(chapter_dir)
chapter_dir.mkdir(parents=True, exist_ok=True)
saved_pdf = chapter_dir / f"{name}.pdf"
shutil.copy2(pdf, saved_pdf)
rep.pdf_path = str(saved_pdf.relative_to(REPO))
rep.pages = pdf_pages(saved_pdf)
text = extract_text(saved_pdf)
rep.leaks = scan_text(text)
rep.image_count = render_images(saved_pdf, chapter_dir / "pages", max_pages)
rep.image_dir = str((chapter_dir / "pages").relative_to(REPO))
return rep
def main():
"""Audit the selected chapters and write the JSON and Markdown reports.
Targets come from ``--fixed``, else ``--all``, else positional
``vol/<chapter>`` tokens (unknown tokens are silently dropped); giving none
is a usage error. Exits 0 only when no chapter leaked or failed to build.
"""
ap = argparse.ArgumentParser()
ap.add_argument("chapters", nargs="*", help="vol/<chap> tokens")
ap.add_argument("--fixed", action="store_true", help="audit only chapters we patched")
ap.add_argument("--all", action="store_true", help="audit every chapter")
ap.add_argument("--max-pages", type=int, default=10,
help="limit pages rendered to PNG; 0 = all")
ap.add_argument("--skip-build", action="store_true")
ap.add_argument("--report", default="audit-pdf-report.json")
ap.add_argument("--text-report", default="audit-pdf-report.md")
args = ap.parse_args()
all_chap = list_all_chapters()
by_key = {f"{v}/{n}": (n, v, q) for (n, v, q) in all_chap}
if args.fixed:
targets = [by_key[f"{v}/{n}"] for (n, v) in FIXED_CHAPTERS if f"{v}/{n}" in by_key]
elif args.all:
targets = all_chap
elif args.chapters:
targets = [by_key[c] for c in args.chapters if c in by_key]
else:
ap.error("must pass chapters, --fixed, or --all")
OUT_DIR.mkdir(exist_ok=True)
print(f"PDF auditing {len(targets)} chapters; max-pages={args.max_pages}", flush=True)
reports: list[PdfReport] = []
for i, (name, vol, qmd) in enumerate(targets, 1):
print(f"[{i:2d}/{len(targets)}] {vol}/{name} ... ", end="", flush=True)
rep = audit_chapter(name, vol, qmd, args.max_pages, build=not args.skip_build)
if not rep.build_ok:
tag = "BUILD FAILED"
elif rep.error:
tag = f"ERROR: {rep.error[:60]}"
else:
counts: dict[str, int] = {}
for L in rep.leaks:
counts[L.pattern] = counts.get(L.pattern, 0) + 1
tag = (f"{len(rep.leaks)} leak(s) {dict(counts)} | "
f"pages={rep.pages} imgs={rep.image_count}")
print(f"{tag} ({rep.build_seconds:.0f}s)", flush=True)
reports.append(rep)
Path(args.report).write_text(json.dumps([asdict(r) for r in reports], indent=2))
lines = ["# PDF Math Rendering Audit", "",
f"Total chapters: **{len(reports)}**", ""]
failed = [r for r in reports if not r.build_ok]
leaky = [r for r in reports if r.build_ok and r.leaks]
clean = [r for r in reports if r.build_ok and not r.leaks and not r.error]
lines += [f"- Clean: **{len(clean)}**",
f"- Leaky: **{len(leaky)}**",
f"- Build failures: **{len(failed)}**", ""]
if failed:
lines += ["## Build failures (likely raw LaTeX outside math mode)", ""]
for r in failed:
lines += [f"### `{r.volume}/{r.name}`",
f"- qmd: `{r.qmd_path}`",
"```", (r.error or "")[-1500:], "```", ""]
if leaky:
lines += ["## Chapters with LaTeX leakage in PDF text", ""]
for r in leaky:
counts: dict[str, int] = {}
for L in r.leaks:
counts[L.pattern] = counts.get(L.pattern, 0) + 1
lines += [f"### `{r.volume}/{r.name}` ({len(r.leaks)} leaks: {dict(counts)})",
f"- pdf: `{r.pdf_path}` ({r.pages} pages)",
f"- images: `{r.image_dir}/` ({r.image_count} png)",
""]
shown: dict[str, int] = {}
for L in r.leaks[:30]:
if shown.get(L.pattern, 0) >= 5:
continue
shown[L.pattern] = shown.get(L.pattern, 0) + 1
lines += [f" - **{L.pattern}** `{L.match}`",
f" > ...{L.context}..."]
lines.append("")
if clean:
lines += ["## Clean chapters (review images for visual confirmation)", ""]
for r in clean:
lines += [f"- `{r.volume}/{r.name}` — pdf `{r.pdf_path}` "
f"({r.pages} pages) — images `{r.image_dir}/` "
f"({r.image_count} png)"]
lines.append("")
Path(args.text_report).write_text("\n".join(lines))
print(f"\nReports: {args.report}, {args.text_report}", flush=True)
print(f"Images: {OUT_DIR}/", flush=True)
print(f"Summary: {len(clean)} clean | {len(leaky)} leaky | "
f"{len(failed)} build-failed", flush=True)
sys.exit(0 if not leaky and not failed else 1)
if __name__ == "__main__":
main()