From 25046065d6a8cc0abf4dfd99aa6c8ef034ecbc07 Mon Sep 17 00:00:00 2001 From: Vijay Janapa Reddi Date: Mon, 8 Jun 2026 16:15:37 -0400 Subject: [PATCH] Polish algorithm pseudocode formatting across vol1 and vol2 Clean up the 13 algorithm blocks so they render well in the narrow text column, in both PDF and HTML: - Inline triangle comments (pdf-right-comment: false) so comments stay glued to their statement instead of stranding at the right margin; set html-comment-delimiter to the triangle so web matches print. - Forbid mid-word hyphenation in the algorithmic environment via a guarded, begin-document-deferred rule in header-includes.tex. - Fix bare \quad / \ spacing macros that broke the HTML pseudocode.js parser (reverse-mode AD, Adam, FlashAttention). - Tighten the few comments/statements that wrapped, and move loop-purpose comments onto the for-headers (tiled GEMM, PTQ, backprop, FlashAttention, power-of-two routing). - Drop manual \textbf from the 1F1B warm-up/drain phase labels. - Gloss the mini-batch SGD prose handoff to describe the loop rather than only announce the algorithm. --- .../contents/vol1/frameworks/frameworks.qmd | 4 +++- .../vol1/hw_acceleration/hw_acceleration.qmd | 4 +++- .../model_compression/model_compression.qmd | 6 ++++-- .../vol1/nn_computation/nn_computation.qmd | 14 +++++++++----- .../quarto/contents/vol1/training/training.qmd | 18 ++++++++++++------ .../collective_communication.qmd | 2 ++ .../distributed_training.qmd | 6 ++++-- .../contents/vol2/inference/inference.qmd | 12 +++++++++--- book/quarto/tex/header-includes.tex | 10 ++++++++++ 9 files changed, 56 insertions(+), 20 deletions(-) diff --git a/book/quarto/contents/vol1/frameworks/frameworks.qmd b/book/quarto/contents/vol1/frameworks/frameworks.qmd index 1e1a2b0126..3503ce6312 100644 --- a/book/quarto/contents/vol1/frameworks/frameworks.qmd +++ b/book/quarto/contents/vol1/frameworks/frameworks.qmd @@ -2307,6 +2307,8 @@ class GPT3ReverseModeParams: ```pseudocode #| label: algo-reverse-mode-ad-trace #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Reverse-mode automatic differentiation} @@ -2316,7 +2318,7 @@ class GPT3ReverseModeParams: \For{$i = 1$ to $n$} \State compute $v_i$, store its value and parent edges \Comment{forward pass} \EndFor -\State $\bar{v}_i \gets 0$ for all $i$; \quad $\bar{y} \gets 1$ +\State $\bar{v}_i \gets 0$ for all $i$; $\bar{y} \gets 1$ \For{$i = n$ down to $1$} \For{each parent $u$ of $v_i$} \State $\bar{u} \gets \bar{u} + \bar{v}_i \, \partial v_i / \partial u$ \Comment{accumulate via the chain rule} diff --git a/book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd b/book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd index 835c286937..67a3780484 100644 --- a/book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd +++ b/book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd @@ -2238,6 +2238,8 @@ To process our `{python} TilingPrinciple.layer_dim_str`-wide layer on a `{python ```pseudocode #| label: algo-tiled-gemm #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Tiled (blocked) matrix multiply} \begin{algorithmic} @@ -2248,7 +2250,7 @@ To process our `{python} TilingPrinciple.layer_dim_str`-wide layer on a `{python \State initialize the $C$-tile accumulator to zero on chip \For{each $k_0$ over $K$ in steps of $T_K$} \State load $A$-tile $[i_0,k_0]$ and $B$-tile $[k_0,j_0]$ on chip - \State accumulate the tile product into the $C$-tile \Comment{each loaded byte reused $T_N$ or $T_M$ times} + \State accumulate the tile product on chip \Comment{reuse $T_N$/$T_M$ per byte} \EndFor \State write the $C$-tile back to memory \EndFor diff --git a/book/quarto/contents/vol1/model_compression/model_compression.qmd b/book/quarto/contents/vol1/model_compression/model_compression.qmd index e4659e4a6e..623d2d7402 100644 --- a/book/quarto/contents/vol1/model_compression/model_compression.qmd +++ b/book/quarto/contents/vol1/model_compression/model_compression.qmd @@ -4051,14 +4051,16 @@ Walk through the PTQ pipeline in @fig-ptq-calibration step by step. A calibratio ```pseudocode #| label: algo-ptq-calibration #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Post-training quantization calibration} \begin{algorithmic} \Require pretrained model $f_\theta$; representative calibration set $C$; bit width $b$; granularity (layer/group/channel); method (max, entropy, percentile) \Ensure quantized weights and per-group range metadata $\{(\alpha_g, \beta_g, s_g, z_g)\}$ \State attach observers to weights and selected activation tensors, at the chosen granularity -\For{each batch in $C$} - \State run $f_\theta$ without updating parameters; record per-group value distributions \Comment{calibration pass} +\For{each batch in $C$} \Comment{calibration pass} + \State run $f_\theta$ in inference; record per-group value distributions \EndFor \For{each quantization group $g$} \State select clipping range $[\alpha_g, \beta_g]$ by the chosen method diff --git a/book/quarto/contents/vol1/nn_computation/nn_computation.qmd b/book/quarto/contents/vol1/nn_computation/nn_computation.qmd index 10ef911333..21ae0120ce 100644 --- a/book/quarto/contents/vol1/nn_computation/nn_computation.qmd +++ b/book/quarto/contents/vol1/nn_computation/nn_computation.qmd @@ -4236,19 +4236,21 @@ The mathematical foundations of backpropagation provide the theoretical basis fo ```pseudocode #| label: algo-backpropagation-training-step #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Backpropagation training step} \begin{algorithmic} \Require mini-batch inputs $\mathbf{X}$, labels $\mathbf{y}$; $N_L$ layers with parameters $\{\mathbf{W}^{(\ell)}, \mathbf{b}^{(\ell)}\}_{\ell=1}^{N_L}$; loss $\mathcal{L}$ \Ensure parameter gradients $\{\nabla_{\mathbf{W}^{(\ell)}}\mathcal{L}, \nabla_{\mathbf{b}^{(\ell)}}\mathcal{L}\}$ for the optimizer \State $\mathbf{A}^{(0)} \gets \mathbf{X}$ -\For{$\ell = 1$ to $N_L$} - \State compute $\mathbf{Z}^{(\ell)}, \mathbf{A}^{(\ell)}$; save the activation and local values its derivative needs \Comment{forward pass} +\For{$\ell = 1$ to $N_L$} \Comment{forward pass} + \State compute $\mathbf{Z}^{(\ell)}, \mathbf{A}^{(\ell)}$; save the values its derivative needs \EndFor \State compute loss $\mathcal{L}(\mathbf{A}^{(N_L)}, \mathbf{y})$; seed the output gradient $\nabla_{\mathbf{A}^{(N_L)}}\mathcal{L}$ -\For{$\ell = N_L$ down to $1$} +\For{$\ell = N_L$ down to $1$} \Comment{same local rule repeats} \State use the saved forward values to compute $\nabla_{\mathbf{W}^{(\ell)}}\mathcal{L}, \nabla_{\mathbf{b}^{(\ell)}}\mathcal{L}$ - \State propagate $\nabla_{\mathbf{A}^{(\ell-1)}}\mathcal{L}$ to the previous layer \Comment{same local rule repeats} + \State propagate $\nabla_{\mathbf{A}^{(\ell-1)}}\mathcal{L}$ to the previous layer \EndFor \State \Return the accumulated parameter gradients \end{algorithmic} @@ -4596,13 +4598,15 @@ class MnistEpochLocal: The complete training process combines forward propagation, backward propagation, and weight updates into a systematic training loop\index{Training Loop!iterative learning}. This loop repeats until the network achieves satisfactory performance or reaches a predetermined number of iterations. -A single pass through the entire training dataset is called an epoch\index{Epoch!dataset iteration}[^fn-epoch-training-nn]. For MNIST, with `{python} MnistEpochLocal.training_examples_str` training images and a batch size of `{python} MnistEpochLocal.batch_size_str`, each epoch consists of `{python} MnistEpochLocal.epoch_batches_str` batch iterations. @Algo-minibatch-sgd summarizes the complete mini-batch SGD procedure\index{Stochastic Gradient Descent!mini-batch procedure}. +A single pass through the entire training dataset is called an epoch\index{Epoch!dataset iteration}[^fn-epoch-training-nn]. For MNIST, with `{python} MnistEpochLocal.training_examples_str` training images and a batch size of `{python} MnistEpochLocal.batch_size_str`, each epoch consists of `{python} MnistEpochLocal.epoch_batches_str` batch iterations. @Algo-minibatch-sgd lays out the mini-batch SGD\index{Stochastic Gradient Descent!mini-batch procedure} loop: each epoch shuffles the data, steps through it one mini-batch at a time with a forward and backward pass, and applies a single parameter update per batch. [^fn-epoch-training-nn]: **Epoch**: One complete pass through all training data. The number of epochs is a direct multiplier on total compute cost: a 100-epoch MNIST run executes 100$\times$ more forward and backward passes than a single epoch. At frontier scale, this multiplier becomes the binding constraint: GPT-3 trained for only ~1 epoch over 300 billion tokens because the per-epoch cost already consumed thousands of GPU-weeks. \index{Epoch!training cycles} ```pseudocode #| label: algo-minibatch-sgd #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Mini-batch stochastic gradient descent} \begin{algorithmic} diff --git a/book/quarto/contents/vol1/training/training.qmd b/book/quarto/contents/vol1/training/training.qmd index 1a94615d1b..bab90747eb 100644 --- a/book/quarto/contents/vol1/training/training.qmd +++ b/book/quarto/contents/vol1/training/training.qmd @@ -1148,6 +1148,8 @@ The `optimizer.step()` method is the other boundary: it consumes the current gra ```pseudocode #| label: algo-adam-update #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Adam parameter update (one optimizer step)} \begin{algorithmic} @@ -1155,7 +1157,7 @@ The `optimizer.step()` method is the other boundary: it consumes the current gra \Ensure updated parameter $\theta_{t+1}$; moment buffers $m_t, v_t$ carried to the next step \State $m_t \gets \beta_1 m_{t-1} + (1-\beta_1)\, g_t$ \Comment{first moment (momentum)} \State $v_t \gets \beta_2 v_{t-1} + (1-\beta_2)\, g_t^2$ \Comment{second moment (variance)} -\State $\hat{m}_t \gets m_t / (1-\beta_1^{t})$; \quad $\hat{v}_t \gets v_t / (1-\beta_2^{t})$ \Comment{bias correction} +\State $\hat{m}_t \gets m_t / (1-\beta_1^{t})$; $\hat{v}_t \gets v_t / (1-\beta_2^{t})$ \Comment{bias correction} \State $\theta_{t+1} \gets \theta_t - \eta\, \hat{m}_t / (\sqrt{\hat{v}_t} + \epsilon)$ \Comment{parameter update} \end{algorithmic} \end{algorithm} @@ -1404,10 +1406,12 @@ To manage these substantial memory requirements, training systems use several so ```pseudocode #| label: algo-gradient-checkpointing #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Gradient checkpointing (activation recomputation)} \begin{algorithmic} -\Require $N_L$-layer network; checkpoint set $\mathcal{C} \subseteq \{1,\dots,N_L\}$ (e.g.\ every $\sqrt{N_L}$ layers) +\Require $N_L$-layer network; checkpoint set $\mathcal{C} \subseteq \{1,\dots,N_L\}$ (e.g. every $\sqrt{N_L}$ layers) \Ensure parameter gradients, at reduced peak activation memory \For{$\ell = 1$ to $N_L$} \State forward layer $\ell$; store its activation only if $\ell \in \mathcal{C}$ \Comment{drop the rest} @@ -4530,21 +4534,23 @@ $$ ```pseudocode #| label: algo-streaming-attention #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{FlashAttention: tiled attention with online softmax} \begin{algorithmic} \Require queries $Q$, keys $K$, values $V$ for a length-$S$ sequence; tile size $b$ \Ensure attention output $Y$, without materializing the $S\times S$ score matrix \For{each query tile $Q_i$} - \State $Y_i^{\text{acc}} \gets \mathbf{0}$; row-wise max $m_i \gets -\infty$; row-wise normalizer $l_i \gets 0$ \Comment{in SRAM} + \State in SRAM: $Y_i^{\text{acc}} \gets \mathbf{0}$, $m_i \gets -\infty$, $l_i \gets 0$ \For{each key/value tile $(K_j, V_j)$} - \State load $Q_i, K_j, V_j$ into SRAM;\ $Z_{ij} \gets Q_i K_j^\top / \sqrt{d_k}$ + \State load $Q_i, K_j, V_j$ into SRAM; $Z_{ij} \gets Q_i K_j^\top / \sqrt{d_k}$ \State $m_i' \gets \max(m_i,\operatorname{rowmax}(Z_{ij}))$ \Comment{new stable max} \State $Y_i^{\text{acc}} \gets Y_i^{\text{acc}}\odot e^{m_i-m_i'} + e^{Z_{ij}-m_i'}V_j$ - \State $l_i \gets l_i\odot e^{m_i-m_i'} + \operatorname{rowsum}(e^{Z_{ij}-m_i'})$;\quad $m_i \gets m_i'$ + \State $l_i \gets l_i\odot e^{m_i-m_i'} + \operatorname{rowsum}(e^{Z_{ij}-m_i'})$; $m_i \gets m_i'$ \State discard $Z_{ij}$ \Comment{never written to HBM} \EndFor - \State $Y_i \gets Y_i^{\text{acc}} / l_i$;\ write $Y_i$ to HBM + \State $Y_i \gets Y_i^{\text{acc}} / l_i$; write $Y_i$ to HBM \EndFor \end{algorithmic} \end{algorithm} diff --git a/book/quarto/contents/vol2/collective_communication/collective_communication.qmd b/book/quarto/contents/vol2/collective_communication/collective_communication.qmd index 0256b6cfe1..1d5b23cd3e 100644 --- a/book/quarto/contents/vol2/collective_communication/collective_communication.qmd +++ b/book/quarto/contents/vol2/collective_communication/collective_communication.qmd @@ -896,6 +896,8 @@ The algorithm splits the vector of size $M$ into $N$ chunks and then alternates ```pseudocode #| label: algo-ring-allreduce #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Ring AllReduce} \begin{algorithmic} diff --git a/book/quarto/contents/vol2/distributed_training/distributed_training.qmd b/book/quarto/contents/vol2/distributed_training/distributed_training.qmd index eb9806cff7..639bee33cb 100644 --- a/book/quarto/contents/vol2/distributed_training/distributed_training.qmd +++ b/book/quarto/contents/vol2/distributed_training/distributed_training.qmd @@ -2137,18 +2137,20 @@ At the schedule level, 1F1B is a warmup, steady-state, and drain loop. @Algo-one ```pseudocode #| label: algo-one-forward-one-backward-pipeline #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{One-forward-one-backward pipeline schedule} \begin{algorithmic} \Require $p$ ordered pipeline stages; $m$ microbatches; activation links $j\to j{+}1$ and gradient links $j{+}1\to j$ \Ensure one optimizer update after gradients accumulate across all $m$ microbatches \State split the global batch into $m$ microbatches; assign consecutive layers to the $p$ stages -\State \textbf{warm-up:} launch forwards from stage 0, forwarding each activation downstream as produced +\State warm-up: launch forwards from stage 0, forwarding each activation downstream as produced \For{each stage in steady state} \State alternate one backward (oldest ready microbatch) with one forward (next microbatch) \State release a microbatch's activation as soon as that stage's backward consumes it \EndFor -\State \textbf{drain:} finish the remaining backward passes in reverse stage order +\State drain: finish the remaining backward passes in reverse stage order \State apply the optimizer update once every stage has accumulated gradients for all $m$ microbatches \end{algorithmic} \end{algorithm} diff --git a/book/quarto/contents/vol2/inference/inference.qmd b/book/quarto/contents/vol2/inference/inference.qmd index c344db1b97..fecbb6ac3a 100644 --- a/book/quarto/contents/vol2/inference/inference.qmd +++ b/book/quarto/contents/vol2/inference/inference.qmd @@ -1012,6 +1012,8 @@ Continuous batching (also called iteration-level batching) decouples batch membe ```pseudocode #| label: algo-continuous-batching-decode-scheduler #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Continuous batching decode scheduler} \begin{algorithmic} @@ -2648,6 +2650,8 @@ Speculative decoding has three operational phases: a Draft phase\index{Speculati ```pseudocode #| label: algo-speculative-decoding #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Speculative decoding} \begin{algorithmic} @@ -3971,15 +3975,17 @@ The procedure is deliberately small, but the load signal must match the serving ```pseudocode #| label: algo-power-two-choices-routing #| pdf-line-number: true +#| pdf-right-comment: false +#| html-comment-delimiter: "▷" \begin{algorithm} \caption{Power-of-two choices request routing} \begin{algorithmic} \Require replica set $\mathcal{S}$; request $r$; capacity weights $w_s$; per-replica work signal $W_s$ (queue length, active tokens, predicted work, or KV-cache occupancy) \Ensure a selected replica for $r$ -\State sample two candidate replicas $s_a, s_b$ from $\mathcal{S}$ \Comment{uniform if homogeneous, capacity-weighted if not} +\State sample two candidate replicas $s_a, s_b$ from $\mathcal{S}$ \Comment{capacity-weighted} \State query $W_{s_a}, W_{s_b}$; estimate incremental work $\hat{c}(r)$ (expected output tokens or KV pages) -\State $u_s \gets (W_s + \hat{c}(r)) / w_s$ for $s \in \{s_a, s_b\}$ \Comment{normalized post-admission load} -\State route $r$ to the candidate with smaller $u_s$ \Comment{ties broken by locality or cache affinity} +\State $u_s \gets (W_s + \hat{c}(r)) / w_s$ for $s \in \{s_a, s_b\}$ \Comment{post-admission load} +\State route $r$ to the candidate with smaller $u_s$ \Comment{ties: cache affinity} \State update the chosen replica's local admission state \end{algorithmic} \end{algorithm} diff --git a/book/quarto/tex/header-includes.tex b/book/quarto/tex/header-includes.tex index 070102e80d..e8b78809b4 100644 --- a/book/quarto/tex/header-includes.tex +++ b/book/quarto/tex/header-includes.tex @@ -63,6 +63,16 @@ labelformat=mylabel,justification=raggedright,singlelinecheck=false,font={ninept % Utility packages \usepackage{etoolbox} % For patching commands and environments +% Algorithm pseudocode (algpseudocodex): in the narrow text column, forbid +% mid-word hyphenation so algorithm steps wrap at spaces, not inside words. +% Deferred to begin-document and guarded so it applies only when a chapter +% actually loads the pseudocode package. +\AtBeginDocument{% + \ifdef{\algorithmic}{% + \AtBeginEnvironment{algorithmic}{\hyphenpenalty=10000\exhyphenpenalty=10000\tolerance=3000\emergencystretch=2.5em}% + }{}% +} + % Page layout and headers \usepackage{fancyhdr} % Custom headers and footers \usepackage{geometry} % Page dimensions and margins