Files
Vijay Janapa Reddi dd29ab9ace vol2: teach Collective Communication before Distributed Training
Distributed Training relies on AllReduce and the other collectives
throughout, so Collective Communication now comes first as chapter 5 and
Distributed Training follows as chapter 6. Folders, printed chapter
numbers, configs, the structure table, StaffML's topic map, the notation
baseline, labs, slides, instructor pages, mlsysim docs, and site
redirects follow. Bridges, part pages, the roadmap, the dependency map,
and the appendix and conclusion tables describe the new order; Collective
Communication glosses the parallelism terms it uses, and Distributed
Training points back to the collectives instead of re-teaching them.
2026-10-13 13:00:24 +02:00

601 lines
22 KiB
Python

#!/usr/bin/env python3
"""Build chapter HTML, audit raw output, and track pass/fail in a ledger.
Usage (repo root)::
python3 binder/tools/audit/chapter_html_verify.py --list
python3 binder/tools/audit/chapter_html_verify.py --vol1 introduction
python3 binder/tools/audit/chapter_html_verify.py --vol1 --all
python3 binder/tools/audit/chapter_html_verify.py --report
Ledger: binder/tools/audit/artifacts/chapter_html_audit.json
Table: binder/tools/audit/artifacts/chapter_html_audit.md
"""
from __future__ import annotations
import argparse
import json
import os
import re
import subprocess
import sys
import time
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
AUDIT_HTML = REPO_ROOT / "binder/tools/audit/fmt/audit_html.py"
LEDGER_JSON = REPO_ROOT / "binder/tools/audit/artifacts/chapter_html_audit.json"
LEDGER_MD = REPO_ROOT / "binder/tools/audit/artifacts/chapter_html_audit.md"
CHAPTERS: dict[str, list[str]] = {
"vol1": [
"introduction/introduction",
"02_system_paradigms/02_system_paradigms",
"ml_lifecycle/ml_lifecycle",
"data_engineering/data_engineering",
"nn_computation/nn_computation",
"nn_architectures/nn_architectures",
"frameworks/frameworks",
"training/training",
"data_selection/data_selection",
"model_compression/model_compression",
"hw_acceleration/hw_acceleration",
"benchmarking/benchmarking",
"model_serving/model_serving",
"ml_ops/ml_ops",
"responsible_engr/responsible_engr",
"conclusion/conclusion",
"backmatter/appendix_algorithm",
"backmatter/appendix_assumptions",
"backmatter/appendix_dam",
"backmatter/appendix_data",
"backmatter/appendix_machine",
],
"vol2": [
"introduction/introduction",
"compute_infrastructure/compute_infrastructure",
"network_fabrics/network_fabrics",
"data_storage/data_storage",
"collective_communication/collective_communication",
"distributed_training/distributed_training",
"fault_tolerance/fault_tolerance",
"fleet_orchestration/fleet_orchestration",
"performance_engineering/performance_engineering",
"inference/inference",
"edge_intelligence/edge_intelligence",
"ops_scale/ops_scale",
"security_privacy/security_privacy",
"robust_ai/robust_ai",
"sustainable_ai/sustainable_ai",
"responsible_ai/responsible_ai",
"conclusion/conclusion",
"backmatter/appendix_dam",
"backmatter/appendix_fleet",
"backmatter/appendix_communication",
"backmatter/appendix_reliability",
"backmatter/appendix_inference",
"backmatter/appendix_c3",
"backmatter/appendix_assumptions",
],
}
HTML_ERROR_PATTERNS = [
re.compile(r"NameError:", re.I),
re.compile(r"KeyError:", re.I),
re.compile(r"AttributeError:", re.I),
re.compile(r"ModuleNotFoundError:", re.I),
re.compile(r"Traceback \(most recent call last\)"),
re.compile(r"Error rendering"),
]
HTML_MATH_RENDER_PATTERNS = [
(
"spaced ASCII multiplication",
re.compile(r"\b\d[\d,]*(?:\.\d+)?\s+x\s+(?:\d|\$|USD|[A-Za-z_(])"),
),
(
"literal dollar-times",
re.compile(r"\$\s*\\times\s*\$"),
),
# 2026-06-07: added after mutation testing — an unclosed `$...` in QMD
# renders as a literal dollar + broken math in reader-visible prose
# ("$() = 1 - (1 - p)^N.") and nothing below caught it. These scan
# _visible_text(), which excludes rendered math spans, so legitimate
# `\(...\)` MathJax sources never match.
(
"unrendered math delimiter",
re.compile(r"\\\(|\\\)|\\\[|\\\]"),
),
(
# `)` deliberately excluded: "(bandwidth/$)" and "per 1M tokens ($)"
# are legitimate currency parentheticals.
"unrendered math: $ before symbol",
re.compile(r"\$\s*[(=\\]"),
),
(
"unresolved crossref",
re.compile(r"\?@(?:sec|fig|tbl|eq|lst|algo)-|(?<![\w./])@(?:sec|fig|tbl|eq|lst|algo)-[\w-]+"),
),
]
@dataclass
class ChapterResult:
"""Ledger record for one chapter's HTML verification run.
Optional boolean check fields stay None when that check did not run.
"""
vol: str
chapter: str
qmd: str
html: str
status: str # pass | fail | pending | skip
build_ok: bool = False
build_seconds: float = 0.0
html_spurious_clean: bool | None = None
html_error_hits: list[str] = field(default_factory=list)
html_math_artifacts_clean: bool | None = None
html_math_artifact_hits: list[str] = field(default_factory=list)
lego_focal_ok: bool | None = None
prose_ok: bool | None = None
registry_ok: bool | None = None
notes: str = ""
updated_at: str = ""
def _chapter_id(vol: str, ch_path: str) -> str:
"""Return the ledger key ``<vol>/<chapter short name>``."""
return f"{vol}/{ch_path.split('/')[-1]}"
def _qmd_path(vol: str, ch_path: str) -> Path:
"""Return the absolute QMD path for a chapter path such as ``training/training``."""
return REPO_ROOT / "books" / vol / f"{ch_path}.qmd"
def _prepare_single_chapter_build(vol: str) -> None:
"""Drop stale chapter HTML so post-render xref scan cannot false-fail.
Fast single-chapter binder builds only re-render the target QMD; leftover
HTML from prior full or partial builds remains under ``html-{vol}/`` and
``verify_rendered_xrefs.py`` scans the whole output tree.
"""
build_dir = REPO_ROOT / "books/_build" / f"html-{vol}"
contents = build_dir / vol
if contents.is_dir():
for html in contents.rglob("*.html"):
html.unlink()
search_json = build_dir / "search.json"
if search_json.is_file():
search_json.unlink()
def _build_html(vol: str, ch_path: str) -> tuple[bool, float, str]:
"""Render one chapter with ``./binder/binder build html``, skipping hygiene and validation.
Deletes stale chapter HTML first, writes combined stdout and stderr to
``/tmp/render_<vol>_<chapter>.log``, and returns
``(succeeded, elapsed_seconds, log_path)``.
"""
name = ch_path.split("/")[-1]
binder_vol = f"--{vol}"
binder_ch = f"{vol}/{name}"
log = Path(f"/tmp/render_{vol}_{name}.log")
_prepare_single_chapter_build(vol)
t0 = time.monotonic()
proc = subprocess.run(
["./binder/binder", "build", "html", binder_vol, binder_ch,
"--skip-hygiene", "--skip-validate"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
)
elapsed = time.monotonic() - t0
log.write_text((proc.stdout or "") + (proc.stderr or ""), encoding="utf-8")
return proc.returncode == 0, elapsed, str(log)
def _live_html(vol: str, ch_path: str) -> Path:
"""Return where the build writes the chapter page, under ``books/_build/html-<vol>/<vol>/``."""
build_dir = REPO_ROOT / "books/_build" / f"html-{vol}" / vol
return build_dir / f"{ch_path}.html"
def _archive_html(vol: str, ch_path: str, live: Path) -> Path:
"""Copy the built page to ``books/_build/html-audit/<vol>/<chapter>.html`` and return the copy."""
name = ch_path.split("/")[-1]
archive_dir = REPO_ROOT / "books/_build/html-audit" / vol
archive_dir.mkdir(parents=True, exist_ok=True)
dest = archive_dir / f"{name}.html"
dest.write_bytes(live.read_bytes())
return dest
def _audit_spurious(html: Path) -> tuple[bool, list[str]]:
"""Run ``fmt/audit_html.py`` on the page to find spurious ``.0`` numbers in prose.
Returns ``(True, [])`` when the script exits 0 and prints ``CLEAN``;
otherwise ``False`` with up to 10 non-blank lines of its output.
"""
proc = subprocess.run(
[sys.executable, str(AUDIT_HTML), str(html)],
capture_output=True,
text=True,
)
if proc.returncode == 0 and "CLEAN" in proc.stdout:
return True, []
issues = [ln for ln in (proc.stdout or proc.stderr).splitlines() if ln.strip()]
return False, issues[:10]
def _scan_html_errors(html: Path) -> list[str]:
"""Return the ``HTML_ERROR_PATTERNS`` that match the page's main content text.
Script, style, pre, and code elements are removed before matching.
Returns an empty list, meaning no errors, when BeautifulSoup is not
installed or the page has no body.
"""
try:
from bs4 import BeautifulSoup
except ImportError:
return []
soup = BeautifulSoup(html.read_text(encoding="utf-8", errors="replace"), "html.parser")
content = soup.find("main") or soup.body
if not content:
return []
for tag in content(["script", "style", "pre", "code"]):
tag.decompose()
text = content.get_text(separator=" ")
hits: list[str] = []
for pat in HTML_ERROR_PATTERNS:
if pat.search(text):
hits.append(pat.pattern)
return hits
def _visible_text(html: Path) -> str:
"""Return the page's reader-visible main content text, whitespace-collapsed.
Removes scripts, styles, code, MathJax annotation and assistive markup, and
client-rendered math and pseudocode containers. Returns an empty string
when BeautifulSoup is not installed or the page has no body.
"""
try:
from bs4 import BeautifulSoup
except ImportError:
return ""
soup = BeautifulSoup(html.read_text(encoding="utf-8", errors="replace"), "html.parser")
content = soup.find("main") or soup.body
if not content:
return ""
for tag in content(["script", "style", "annotation", "mjx-assistive-mml", "pre", "code"]):
tag.decompose()
# Rendered math spans legitimately contain raw `\(...\)` source (MathJax
# processes client-side), and pseudocode blocks contain raw algorithmic
# LaTeX (pseudocode.js renders client-side too); exclude both so the
# unrendered-math patterns only fire on LaTeX that leaked OUTSIDE a
# client-rendered container. (2026-06-07)
for tag in content.find_all(class_=re.compile(r"\bmath\b|pseudocode")):
tag.decompose()
return re.sub(r"\s+", " ", content.get_text(separator=" "))
def _scan_math_render_artifacts(html: Path) -> list[str]:
"""Scan visible text for math-rendering artifacts and unresolved cross-references.
Returns ``"<label>: <context>"`` strings with up to 50 characters of
context on each side of the match, capped at 10 hits.
"""
text = _visible_text(html)
if not text:
return []
hits: list[str] = []
for label, pat in HTML_MATH_RENDER_PATTERNS:
for match in pat.finditer(text):
start = max(0, match.start() - 50)
end = min(len(text), match.end() + 50)
context = text[start:end].strip()
hits.append(f"{label}: {context}")
if len(hits) >= 10:
return hits
return hits
def _lego_focal(qmd: Path) -> tuple[bool, str]:
"""Run ``lego_focal_verify.py`` on the QMD and return ``(ok, message)``.
Passes on a zero exit with no output, or a zero exit whose output contains
``issues=0``. The message is the last output line.
"""
proc = subprocess.run(
[sys.executable, str(REPO_ROOT / "binder/tools/audit/lego_focal_verify.py"), str(qmd)],
cwd=REPO_ROOT,
capture_output=True,
text=True,
)
out = (proc.stdout or "") + (proc.stderr or "")
if proc.returncode == 0 and not out.strip():
return True, "no inline Python refs"
ok = proc.returncode == 0 and "issues=0" in out
return ok, out.strip().splitlines()[-1] if out.strip() else "no output"
def _prose_exec(qmd: Path, timeout_s: int = 120) -> tuple[bool, str]:
"""Execute the chapter's cells and prose previews via ``fmt/audit_prose.py``.
Runs with ``mlsysim`` on ``PYTHONPATH`` and the Agg matplotlib backend.
Returns ``(True, "ok")`` on a zero exit; otherwise ``False`` with the first
300 characters of error output, or a timeout message after ``timeout_s``.
"""
env = {**os.environ, "PYTHONPATH": str(REPO_ROOT / "mlsysim"), "MPLBACKEND": "Agg"}
try:
proc = subprocess.run(
[sys.executable, "-W", "ignore::UserWarning",
str(REPO_ROOT / "binder/tools/audit/fmt/audit_prose.py"), str(qmd)],
cwd=REPO_ROOT,
env=env,
capture_output=True,
text=True,
timeout=timeout_s,
)
except subprocess.TimeoutExpired:
return False, f"prose exec timed out after {timeout_s}s"
if proc.returncode != 0:
err = (proc.stderr or proc.stdout or "prose exec failed").strip()
return False, err[:300]
return True, "ok"
def _registry_scan(qmd: Path) -> tuple[bool, str]:
"""Run ``book_check_registry_sources.py`` on the QMD; return (exit 0, last 200 chars of output)."""
proc = subprocess.run(
[sys.executable, str(REPO_ROOT / "binder/tools/audit/book_check_registry_sources.py"), str(qmd)],
cwd=REPO_ROOT,
capture_output=True,
text=True,
)
return proc.returncode == 0, (proc.stdout or proc.stderr or "").strip()[-200:]
def verify_chapter(vol: str, ch_path: str, skip_build: bool = False) -> ChapterResult:
"""Run every check for one chapter and return its ledger record.
Runs the registry source scan, LEGO focal check, and prose execution, then
builds and archives the chapter HTML (or, with ``skip_build``, reuses the
archived copy) and scans it for error text, math-rendering artifacts, and
spurious ``.0`` numbers. Failure details accumulate in ``notes``. Status is
``skip`` when the QMD is missing, ``pending`` when ``skip_build`` finds no
archive, ``fail`` on a build failure, missing output, or any failed check,
and ``pass`` otherwise.
"""
qmd = _qmd_path(vol, ch_path)
name = ch_path.split("/")[-1]
res = ChapterResult(
vol=vol,
chapter=name,
qmd=str(qmd.relative_to(REPO_ROOT)),
html="",
status="pending",
updated_at=datetime.now(timezone.utc).isoformat(),
)
if not qmd.is_file():
res.status = "skip"
res.notes = "QMD missing"
return res
reg_ok, reg_msg = _registry_scan(qmd)
res.registry_ok = reg_ok
if not reg_ok:
res.notes += f"registry: {reg_msg}; "
lego_ok, lego_msg = _lego_focal(qmd)
res.lego_focal_ok = lego_ok
if not lego_ok:
res.notes += f"lego: {lego_msg}; "
prose_ok, prose_msg = _prose_exec(qmd)
res.prose_ok = prose_ok
if not prose_ok:
res.notes += f"prose: {prose_msg}; "
if skip_build:
archive = REPO_ROOT / "books/_build/html-audit" / vol / f"{name}.html"
if not archive.is_file():
res.status = "pending"
res.notes += "no archived HTML (--skip-build)"
return res
html = archive
res.build_ok = True
res.build_seconds = 0.0
else:
build_ok, secs, log = _build_html(vol, ch_path)
res.build_ok = build_ok
res.build_seconds = round(secs, 1)
if not build_ok:
res.status = "fail"
res.notes += f"build failed ({log}); "
return res
live = _live_html(vol, ch_path)
if not live.is_file():
res.status = "fail"
res.notes += f"HTML missing at {live}; "
return res
html = _archive_html(vol, ch_path, live)
res.html = str(html.relative_to(REPO_ROOT))
res.html_error_hits = _scan_html_errors(html)
res.html_math_artifact_hits = _scan_math_render_artifacts(html)
res.html_math_artifacts_clean = not res.html_math_artifact_hits
clean, spurious = _audit_spurious(html)
res.html_spurious_clean = clean
if res.html_error_hits:
res.notes += f"html errors: {res.html_error_hits}; "
if res.html_math_artifact_hits:
res.notes += f"math artifacts: {res.html_math_artifact_hits[:2]}; "
if not clean:
res.notes += f"spurious .0: {spurious[:2]}; "
passed = (
res.build_ok
and not res.html_error_hits
and res.html_math_artifacts_clean
and res.html_spurious_clean
and res.lego_focal_ok
and res.prose_ok
and res.registry_ok
)
res.status = "pass" if passed else "fail"
if passed:
res.notes = "OK — build + HTML clean + LEGO/registry consistent"
return res
def load_ledger() -> dict:
"""Load the JSON ledger, or return an empty ledger if the file does not exist."""
if LEDGER_JSON.is_file():
return json.loads(LEDGER_JSON.read_text(encoding="utf-8"))
return {"updated_at": "", "chapters": {}}
def save_ledger(ledger: dict) -> None:
"""Stamp ``updated_at`` on the ledger, write the JSON, and regenerate the Markdown table."""
LEDGER_JSON.parent.mkdir(parents=True, exist_ok=True)
ledger["updated_at"] = datetime.now(timezone.utc).isoformat()
LEDGER_JSON.write_text(json.dumps(ledger, indent=2), encoding="utf-8")
_write_markdown_table(ledger)
def _write_markdown_table(ledger: dict) -> None:
"""Write the ledger to ``LEDGER_MD`` as a status table with pass/fail/pending totals."""
rows = []
for key in sorted(ledger.get("chapters", {}).keys()):
c = ledger["chapters"][key]
rows.append(c)
def yn(v):
"""Map True, False, or anything else to a check, cross, or dash cell."""
if v is True:
return "✅"
if v is False:
return "❌"
return "—"
lines = [
"# Chapter HTML audit ledger",
"",
f"Updated: {ledger.get('updated_at', '')}",
"",
"Registry migration chapter verification: build HTML → raw HTML scan → LEGO/prose/registry checks.",
"",
"| Vol | Chapter | Status | Build | HTML | Math | Spurious | LEGO | Prose | Registry | s | Notes |",
"|-----|---------|--------|-------|------|------|----------|------|-------|----------|---|-------|",
]
pass_n = fail_n = pending_n = 0
for c in rows:
st = c.get("status", "pending")
if st == "pass":
pass_n += 1
elif st == "fail":
fail_n += 1
else:
pending_n += 1
notes = (c.get("notes") or "").replace("|", "\\|")[:80]
lines.append(
f"| {c['vol']} | {c['chapter']} | **{st}** | {yn(c.get('build_ok'))} | "
f"{yn(not c.get('html_error_hits')) if c.get('html') else '—'} | "
f"{yn(c.get('html_math_artifacts_clean'))} | "
f"{yn(c.get('html_spurious_clean'))} | {yn(c.get('lego_focal_ok'))} | "
f"{yn(c.get('prose_ok'))} | {yn(c.get('registry_ok'))} | "
f"{c.get('build_seconds', '')} | {notes} |"
)
lines.extend([
"",
f"**Summary:** {pass_n} pass / {fail_n} fail / {pending_n} pending / {len(rows)} total",
"",
"Re-run one chapter: `python3 binder/tools/audit/chapter_html_verify.py --vol1 training`",
"",
])
LEDGER_MD.write_text("\n".join(lines), encoding="utf-8")
def main() -> int:
"""Command-line entry point.
``--list`` prints chapter names and ``--report`` only regenerates the
Markdown table. Otherwise verifies the chapters named after ``--vol1`` or
``--vol2``, saving the ledger after each one. ``--all`` with a bare volume
flag, or with none, selects every chapter of both volumes; with named
chapters it selects every chapter of the flagged volume. Returns 0 when no
chapter fails, 1 if any fails, and 2 (after printing help) when nothing is
selected; unknown chapter names exit through ``parser.error``.
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--vol1", nargs="*", metavar="CHAPTER", help="Vol1 chapter(s) by short name")
parser.add_argument("--vol2", nargs="*", metavar="CHAPTER", help="Vol2 chapter(s) by short name")
parser.add_argument("--all", action="store_true", help="All chapters in selected volume(s)")
parser.add_argument("--list", action="store_true", help="List chapters")
parser.add_argument("--report", action="store_true", help="Regenerate markdown table only")
parser.add_argument("--skip-build", action="store_true", help="Audit archived HTML only")
args = parser.parse_args()
if args.list:
for vol, paths in CHAPTERS.items():
print(f"\n{vol}:")
for p in paths:
print(f" {p.split('/')[-1]}")
return 0
if args.report:
save_ledger(load_ledger())
print(f"Wrote {LEDGER_MD}")
return 0
targets: list[tuple[str, str]] = []
if args.all:
if args.vol1 is not None or (not args.vol1 and not args.vol2):
targets.extend(("vol1", p) for p in CHAPTERS["vol1"])
if args.vol2 is not None or (not args.vol1 and not args.vol2):
targets.extend(("vol2", p) for p in CHAPTERS["vol2"])
else:
for vol_flag, vol in [(args.vol1, "vol1"), (args.vol2, "vol2")]:
if vol_flag is None:
continue
names = vol_flag if vol_flag else []
if not names:
parser.error(f"Provide chapter names or --all for {vol}")
path_map = {p.split("/")[-1]: p for p in CHAPTERS[vol]}
for name in names:
if name not in path_map:
parser.error(f"Unknown {vol} chapter: {name}")
targets.append((vol, path_map[name]))
if not targets:
parser.print_help()
return 2
ledger = load_ledger()
ledger.setdefault("chapters", {})
fail = 0
for vol, ch_path in targets:
cid = _chapter_id(vol, ch_path)
print(f"\n=== {cid} ===")
result = verify_chapter(vol, ch_path, skip_build=args.skip_build)
ledger["chapters"][cid] = asdict(result)
save_ledger(ledger)
print(f" {result.status.upper()} ({result.build_seconds}s) — {result.notes[:120]}")
if result.status == "fail":
fail += 1
print(f"\nLedger: {LEDGER_JSON}")
print(f"Table: {LEDGER_MD}")
return 1 if fail else 0
if __name__ == "__main__":
raise SystemExit(main())