From f4f69c8a53c2fb8e549ebed01a39fe216b0bbfd9 Mon Sep 17 00:00:00 2001
From: Vijay Janapa Reddi <vj@eecs.harvard.edu>
Date: Tue, 28 Apr 2026 18:13:03 -0400
Subject: [PATCH] feat(ci): visual smoke gate before tinytorch dev deploy
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a Playwright-based smoke check that runs after the Quarto build
and before the dev preview deploy. Catches the four classes of
regression we just fixed in commit 6fdf81dd4:

  • <link rel=stylesheet> 404s (broken site_libs/ deploy)
  • JS console errors during page load
  • Blank-page renders (homepage body shorter than 1.5× viewport)
  • Navbar collapse breakpoint drift across the shared chrome (asserts
    expanded at 1200/1400, collapsed at 992/1199 in light + dark)

Layout follows the publish-guard pattern already in use:

  .github/scripts/visual_smoke.py        — test logic, runnable locally
                                           against any Quarto _build/ tree
  .github/workflows/infra-visual-smoke.yml
                                         — workflow_call reusable that
                                           runs the script on an uploaded
                                           build artifact

tinytorch-preview-dev.yml is split into three jobs:
  1. build-site  — builds Quarto, injects PDFs, uploads _build/ artifact
  2. smoke       — `uses:` the reusable; deploy is gated on this passing
  3. deploy      — downloads the same artifact (so we ship the exact
                   bytes smoke validated) and SSH-pushes to the dev repo

Verified locally:
  • script passes 8/8 (4 viewports × 2 schemes) on the fixed dev tip
  • script catches the regression: with collapse-below reverted to "lg"
    and re-rendered, smoke fails 4/8 with [NAVBAR_COLLAPSE_AT_XL] at
    exactly 992 + 1199 px in light + dark — the same widths a human
    would notice the wrap

Other sites' preview-dev workflows are intentionally untouched. Once
this proof-of-concept settles, copying the build-site/smoke/deploy
split to labs/kits/mlsysim/site/etc. is mechanical (the reusable
workflow doesn't change). A follow-up infra-visual-cross-site.yml
will run nightly across every site in a matrix and add the cross-site
consistency assertions.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .github/scripts/visual_smoke.py             | 252 ++++++++++++++++++++
 .github/workflows/infra-visual-smoke.yml    | 103 ++++++++
 .github/workflows/tinytorch-preview-dev.yml |  47 +++-
 3 files changed, 398 insertions(+), 4 deletions(-)
 create mode 100755 .github/scripts/visual_smoke.py
 create mode 100644 .github/workflows/infra-visual-smoke.yml

diff --git a/.github/scripts/visual_smoke.py b/.github/scripts/visual_smoke.py
new file mode 100755
index 0000000000..a1d83eeab9
--- /dev/null
+++ b/.github/scripts/visual_smoke.py
@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+# =============================================================================
+# Visual Smoke Test — Quarto site rendering sanity check
+# =============================================================================
+# Runs four cheap assertions against a built _build/ directory, at four
+# viewport widths × light + dark color schemes, on each --page passed in:
+#
+#   1. STYLESHEETS_RESOLVE — every <link rel="stylesheet"> returns 200.
+#      Catches the site_libs/ deploy regression that left mlsysbook.ai/about/
+#      shipping a raw <ul> navbar with no Bootstrap (commit 6fdf81dd4).
+#
+#   2. NO_CONSOLE_ERRORS — Playwright's console listener captures zero
+#      severity=error messages during load. Catches JS failures from
+#      missing CDNs, broken theme-bridge scripts, and similar.
+#
+#   3. PAGE_HAS_HEIGHT — body.scrollHeight > 1.5 × viewport_h on the
+#      homepage. The about-page failure mode rendered at body height
+#      ~equal to the viewport because all the styled content was stacked
+#      below 3000 px of unstyled whitespace; this assertion bottoms out
+#      that class of regression.
+#
+#   4. NAVBAR_COLLAPSE_AT_XL — at widths ≥ 1200 px the .navbar-collapse
+#      element should be visible (no hamburger); at widths ≤ 1199 px it
+#      should be hidden (hamburger only). This is the shared-chrome
+#      breakpoint that issues #1 and #2 broke; one assertion proves the
+#      shared navbar is consistent across every site that consumes
+#      shared/config/navbar-common.yml.
+#
+# Usage:
+#   visual_smoke.py --build-dir tinytorch/quarto/_build --site tinytorch
+#                   --pages /index.html /preface.html
+#                   [--report-dir _smoke-report] [--port 0]
+#
+# Exit codes:
+#   0  all assertions passed
+#   1  one or more assertions failed (see report-dir/results.json)
+#   2  setup error (Playwright not installed, build dir missing, etc.)
+# =============================================================================
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+try:
+    from playwright.sync_api import sync_playwright
+except ImportError:
+    print("ERROR: playwright not installed. Run: pip install playwright && playwright install chromium", file=sys.stderr)
+    sys.exit(2)
+
+
+# Viewport widths chosen to bracket the navbar's xl collapse breakpoint:
+# 1199 → just below collapse, must be hamburger; 1200 → just at, must be expanded.
+VIEWPORTS: list[tuple[int, int]] = [(1400, 900), (1200, 900), (1199, 900), (992, 900)]
+COLOR_SCHEMES: list[str] = ["light", "dark"]
+NAVBAR_EXPANDED_MIN_WIDTH = 1200  # matches collapse-below: xl in navbar-common.yml
+
+
+@dataclass
+class Failure:
+    """One assertion that failed. Aggregated and reported at the end."""
+    page: str
+    viewport: str
+    scheme: str
+    assertion: str
+    detail: str
+
+
+@dataclass
+class Report:
+    site: str
+    failures: list[Failure] = field(default_factory=list)
+    pages_checked: int = 0
+    matrix_runs: int = 0  # pages × viewports × schemes
+
+    def add(self, page: str, vp: tuple[int, int], scheme: str, assertion: str, detail: str) -> None:
+        self.failures.append(Failure(page, f"{vp[0]}x{vp[1]}", scheme, assertion, detail))
+
+    def to_dict(self) -> dict:
+        return {
+            "site": self.site,
+            "pages_checked": self.pages_checked,
+            "matrix_runs": self.matrix_runs,
+            "failure_count": len(self.failures),
+            "failures": [f.__dict__ for f in self.failures],
+        }
+
+
+def _free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+@contextlib.contextmanager
+def _serve(directory: Path, port: int):
+    """Boot a SimpleHTTPServer rooted at directory; teardown on exit."""
+    handler = lambda *a, **kw: SimpleHTTPRequestHandler(*a, directory=str(directory), **kw)  # noqa: E731
+    server = ThreadingHTTPServer(("127.0.0.1", port), handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    # Tiny wait so the first request doesn't race the bind().
+    time.sleep(0.1)
+    try:
+        yield f"http://127.0.0.1:{port}"
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+def _check_page(page, base_url: str, page_path: str, vp: tuple[int, int], scheme: str, report: Report) -> None:
+    """Run all four assertions on one (page, viewport, scheme) tuple."""
+    url = base_url.rstrip("/") + page_path
+    console_errors: list[str] = []
+    page.on("console", lambda msg: console_errors.append(msg.text) if msg.type == "error" else None)
+
+    page.goto(url, wait_until="networkidle", timeout=30000)
+
+    # 1. STYLESHEETS_RESOLVE — every <link rel=stylesheet> returns 200.
+    css_results = page.evaluate(
+        """async () => {
+          const links = Array.from(document.querySelectorAll('link[rel~="stylesheet"]'));
+          const out = [];
+          for (const l of links) {
+            try {
+              const r = await fetch(l.href, { method: 'HEAD' });
+              out.push({ href: l.href, status: r.status });
+            } catch (e) {
+              out.push({ href: l.href, status: 0, error: String(e) });
+            }
+          }
+          return out;
+        }"""
+    )
+    bad_css = [c for c in css_results if c["status"] != 200]
+    if bad_css:
+        report.add(page_path, vp, scheme, "STYLESHEETS_RESOLVE",
+                   f"{len(bad_css)} stylesheet(s) failed: " +
+                   ", ".join(f"{c['href']}→{c['status']}" for c in bad_css[:3]))
+
+    # 2. NO_CONSOLE_ERRORS — capture window for the page load. (Listener was
+    # attached above; any errors during the goto/networkidle window are now
+    # in console_errors.)
+    if console_errors:
+        report.add(page_path, vp, scheme, "NO_CONSOLE_ERRORS",
+                   f"{len(console_errors)} console error(s): " +
+                   "; ".join(console_errors[:2]))
+
+    # 3. PAGE_HAS_HEIGHT — body taller than 1.5× viewport. We only enforce
+    # this on the homepage of each site to avoid false positives on
+    # intentionally short pages (404s, redirect stubs).
+    if page_path in ("/", "/index.html"):
+        body_h = page.evaluate("document.body.scrollHeight")
+        if body_h < vp[1] * 1.5:
+            report.add(page_path, vp, scheme, "PAGE_HAS_HEIGHT",
+                       f"body.scrollHeight={body_h}px < {int(vp[1] * 1.5)}px (viewport_h × 1.5)")
+
+    # 4. NAVBAR_COLLAPSE_AT_XL — collapse boundary is consistent.
+    nav_state = page.evaluate(
+        """() => {
+          const collapse = document.querySelector('.navbar-collapse');
+          const toggler = document.querySelector('.navbar-toggler');
+          if (!collapse || !toggler) return null;
+          // Bootstrap toggles `display: none` on the toggler at expanded
+          // breakpoints. That's a more reliable signal than .navbar-collapse
+          // visibility, which Bootstrap always renders (it's the *expanded*
+          // collapse-target, just with `.collapse` toggled).
+          const togglerVisible = getComputedStyle(toggler).display !== 'none';
+          return { togglerVisible };
+        }"""
+    )
+    if nav_state is not None:
+        expected_collapsed = vp[0] < NAVBAR_EXPANDED_MIN_WIDTH
+        actually_collapsed = nav_state["togglerVisible"]
+        if expected_collapsed != actually_collapsed:
+            report.add(page_path, vp, scheme, "NAVBAR_COLLAPSE_AT_XL",
+                       f"width={vp[0]}: expected collapsed={expected_collapsed}, "
+                       f"actual collapsed={actually_collapsed} (toggler visible={nav_state['togglerVisible']})")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run visual smoke checks against a built Quarto site.")
+    parser.add_argument("--build-dir", type=Path, required=True, help="Path to _build/ directory")
+    parser.add_argument("--site", required=True, help="Site name (for the report)")
+    parser.add_argument("--pages", nargs="+", default=["/index.html"], help="Page paths to check (default: /index.html)")
+    parser.add_argument("--report-dir", type=Path, default=Path("_smoke-report"), help="Where to write results.json + failure screenshots")
+    parser.add_argument("--port", type=int, default=0, help="Port to serve on (0 = pick free port)")
+    args = parser.parse_args()
+
+    if not args.build_dir.is_dir():
+        print(f"ERROR: --build-dir {args.build_dir} does not exist or is not a directory", file=sys.stderr)
+        return 2
+
+    args.report_dir.mkdir(parents=True, exist_ok=True)
+    report = Report(site=args.site)
+    port = args.port or _free_port()
+
+    print(f"🔎 visual smoke: site={args.site} build={args.build_dir} pages={args.pages}")
+
+    with _serve(args.build_dir, port) as base_url:
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            try:
+                for page_path in args.pages:
+                    report.pages_checked += 1
+                    for vp in VIEWPORTS:
+                        for scheme in COLOR_SCHEMES:
+                            ctx = browser.new_context(
+                                viewport={"width": vp[0], "height": vp[1]},
+                                color_scheme=scheme,
+                            )
+                            page = ctx.new_page()
+                            try:
+                                _check_page(page, base_url, page_path, vp, scheme, report)
+                            except Exception as e:
+                                report.add(page_path, vp, scheme, "PAGE_LOAD",
+                                           f"{type(e).__name__}: {e}")
+                                # Snapshot the broken state so the failure is debuggable.
+                                with contextlib.suppress(Exception):
+                                    snap = args.report_dir / f"fail_{page_path.strip('/').replace('/', '_') or 'index'}_{vp[0]}_{scheme}.png"
+                                    page.screenshot(path=str(snap), full_page=False)
+                            finally:
+                                ctx.close()
+                            report.matrix_runs += 1
+            finally:
+                browser.close()
+
+    results_path = args.report_dir / "results.json"
+    results_path.write_text(json.dumps(report.to_dict(), indent=2))
+
+    if report.failures:
+        print(f"\n❌ {len(report.failures)} failure(s) across {report.matrix_runs} runs:")
+        for f in report.failures:
+            print(f"   [{f.assertion}] {f.page} @ {f.viewport} {f.scheme}: {f.detail}")
+        print(f"\nResults: {results_path}")
+        return 1
+
+    print(f"\n✅ {report.matrix_runs} runs across {report.pages_checked} page(s), all green.")
+    print(f"Results: {results_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/workflows/infra-visual-smoke.yml b/.github/workflows/infra-visual-smoke.yml
new file mode 100644
index 0000000000..c90a9e0972
--- /dev/null
+++ b/.github/workflows/infra-visual-smoke.yml
@@ -0,0 +1,103 @@
+name: '🔧 Infra · 👁️ Visual Smoke'
+
+# =============================================================================
+# 👁️ Visual Smoke — Reusable Quarto-site rendering check
+# =============================================================================
+#
+# Reusable workflow called from each <site>-preview-dev.yml after the site
+# is built. Runs .github/scripts/visual_smoke.py against an uploaded
+# _build/ artifact, asserting:
+#
+#   • Every <link rel=stylesheet> returns 200 (catches deploy regressions
+#     that drop site_libs/ — see commit 6fdf81dd4).
+#   • No JS console errors during page load.
+#   • Homepage body height > 1.5× viewport (catches blank-page failures
+#     where CSS fails and content stacks below 3000 px of whitespace).
+#   • Navbar collapse-state matches the shared collapse-below: xl
+#     breakpoint at 992 / 1199 / 1200 / 1400 px in light + dark.
+#
+# Inputs:
+#   site         — site name (used in the report; e.g. "tinytorch").
+#   artifact     — name of the workflow-artifact containing the _build/ tree
+#                  the caller uploaded earlier.
+#   pages        — space-separated page paths to check, default "/index.html".
+#                  Each site can extend this with site-specific pages.
+#
+# Outputs:
+#   none. The job exits non-zero on any assertion failure, gating the caller.
+#
+# Failure debugging:
+#   On failure the smoke report (results.json + any failure screenshots) is
+#   uploaded as the "<site>-visual-smoke-report" artifact for inspection.
+#
+# Related:
+#   - .github/scripts/visual_smoke.py — actual test logic (runnable locally).
+#   - tinytorch-preview-dev.yml — first caller (proof-of-concept).
+#
+# =============================================================================
+
+on:
+  workflow_call:
+    inputs:
+      site:
+        description: 'Site name (e.g. tinytorch). Used for the report only.'
+        required: true
+        type: string
+      artifact:
+        description: 'Name of the build artifact containing _build/ tree.'
+        required: true
+        type: string
+      pages:
+        description: 'Space-separated page paths to check (default: /index.html).'
+        required: false
+        type: string
+        default: '/index.html'
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  smoke:
+    name: '👁️ Visual smoke (${{ inputs.site }})'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: 📥 Checkout (script + shared config)
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: |
+            .github/scripts/visual_smoke.py
+
+      - name: 📦 Download build artifact
+        uses: actions/download-artifact@v8
+        with:
+          name: ${{ inputs.artifact }}
+          path: _build
+
+      - name: 🐍 Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: 🎭 Install Playwright + Chromium
+        run: |
+          pip install playwright
+          playwright install --with-deps chromium
+
+      - name: 👁️ Run visual smoke
+        run: |
+          python .github/scripts/visual_smoke.py \
+            --build-dir _build \
+            --site "${{ inputs.site }}" \
+            --pages ${{ inputs.pages }} \
+            --report-dir _smoke-report
+
+      - name: 📤 Upload smoke report on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.site }}-visual-smoke-report
+          path: _smoke-report/
+          retention-days: 7
diff --git a/.github/workflows/tinytorch-preview-dev.yml b/.github/workflows/tinytorch-preview-dev.yml
index d899b90bb9..e0d42ac99f 100644
--- a/.github/workflows/tinytorch-preview-dev.yml
+++ b/.github/workflows/tinytorch-preview-dev.yml
@@ -49,9 +49,11 @@ jobs:
     if: (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'dev') || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/tinytorch-build-pdfs.yml
 
-  # Build and deploy (after PDFs complete - deploys even if PDFs fail)
-  build-and-deploy:
-    name: '🔥 Build & Deploy TinyTorch Dev'
+  # Build the site (after PDFs complete — site builds even if PDFs fail).
+  # Split out from deploy so the artifact gates the visual-smoke job before
+  # any dev preview goes live; deploy only runs if smoke is green.
+  build-site:
+    name: '🔥 Build TinyTorch Dev Site'
     runs-on: ubuntu-latest
     needs: [build-pdfs]
     if: |
@@ -147,6 +149,43 @@ jobs:
             --commit-hash "${{ github.sha }}" \
             --commit-short "$COMMIT_SHORT"
 
+      - name: 📤 Upload built site as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: tinytorch-dev-build
+          path: ${{ vars.TINYTORCH_SITE }}/_build
+          retention-days: 1
+          include-hidden-files: true
+
+  # Visual smoke gate — fails before deploy if Bootstrap/CSS regressed,
+  # the navbar collapse breakpoint shifted, or pages render blank. Runs
+  # the shared .github/scripts/visual_smoke.py against the artifact above.
+  smoke:
+    name: '👁️ Visual Smoke'
+    needs: [build-site]
+    if: needs.build-site.result == 'success'
+    uses: ./.github/workflows/infra-visual-smoke.yml
+    with:
+      site: tinytorch
+      artifact: tinytorch-dev-build
+      pages: '/index.html /preface.html'
+
+  # Deploy is gated on smoke passing. Re-fetches the artifact rather than
+  # the raw repo so we deploy the exact bytes that smoke validated.
+  deploy:
+    name: '🚀 Deploy TinyTorch Dev'
+    runs-on: ubuntu-latest
+    needs: [smoke]
+    if: needs.smoke.result == 'success'
+    permissions:
+      contents: read
+    steps:
+      - name: 📦 Download built site
+        uses: actions/download-artifact@v8
+        with:
+          name: tinytorch-dev-build
+          path: _build
+
       - name: 🚀 Deploy to Dev Site via SSH
         env:
           SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
@@ -171,7 +210,7 @@ jobs:
           mkdir -p ${{ vars.DEV_TINYTORCH_PATH }}
 
           echo "🚚 Copying TinyTorch site content..."
-          cp -r "${{ github.workspace }}/${{ vars.TINYTORCH_SITE }}/_build/." ${{ vars.DEV_TINYTORCH_PATH }}/
+          cp -r "${{ github.workspace }}/_build/." ${{ vars.DEV_TINYTORCH_PATH }}/
 
           echo "🔍 Validating deployment content..."
           if [ ! -f "${{ vars.DEV_TINYTORCH_PATH }}/index.html" ]; then