github-starred/cs249r_book
2
0
Fork 0
mirror of https://github.com/harvard-edge/cs249r_book.git synced 2026-07-21 17:57:24 -05:00
Code Issues 276 Packages Projects Releases 26 Wiki Activity

Polish algorithm pseudocode formatting across vol1 and vol2

Browse Source 
Clean up the 13 algorithm blocks so they render well in the narrow text
column, in both PDF and HTML:

- Inline triangle comments (pdf-right-comment: false) so comments stay glued
  to their statement instead of stranding at the right margin; set
  html-comment-delimiter to the triangle so web matches print.
- Forbid mid-word hyphenation in the algorithmic environment via a guarded,
  begin-document-deferred rule in header-includes.tex.
- Fix bare \quad / \  spacing macros that broke the HTML pseudocode.js parser
  (reverse-mode AD, Adam, FlashAttention).
- Tighten the few comments/statements that wrapped, and move loop-purpose
  comments onto the for-headers (tiled GEMM, PTQ, backprop, FlashAttention,
  power-of-two routing).
- Drop manual \textbf from the 1F1B warm-up/drain phase labels.
- Gloss the mini-batch SGD prose handoff to describe the loop rather than only
  announce the algorithm.
This commit is contained in:
Vijay Janapa Reddi
2026-06-08 16:15:37 -04:00
parent 53e9e232c6
commit 25046065d6
9 changed files with 56 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
book/quarto/contents/vol1/frameworks/frameworks.qmd
+3 -1
View File
@@ -2307,6 +2307,8 @@ class GPT3ReverseModeParams:
```pseudocode
#| label: algo-reverse-mode-ad-trace
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Reverse-mode automatic differentiation}
@@ -2316,7 +2318,7 @@ class GPT3ReverseModeParams:
\For{$i = 1$ to $n$}
\State compute $v_i$, store its value and parent edges \Comment{forward pass}
\EndFor
\State $\bar{v}_i \gets 0$ for all $i$; \quad $\bar{y} \gets 1$
\State $\bar{v}_i \gets 0$ for all $i$; $\bar{y} \gets 1$
\For{$i = n$ down to $1$}
\For{each parent $u$ of $v_i$}
\State $\bar{u} \gets \bar{u} + \bar{v}_i \, \partial v_i / \partial u$ \Comment{accumulate via the chain rule}
book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd
+3 -1
View File
@@ -2238,6 +2238,8 @@ To process our `{python} TilingPrinciple.layer_dim_str`-wide layer on a `{python
```pseudocode
#| label: algo-tiled-gemm
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Tiled (blocked) matrix multiply}
\begin{algorithmic}
@@ -2248,7 +2250,7 @@ To process our `{python} TilingPrinciple.layer_dim_str`-wide layer on a `{python
\State initialize the $C$-tile accumulator to zero on chip
\For{each $k_0$ over $K$ in steps of $T_K$}
\State load $A$-tile $[i_0,k_0]$ and $B$-tile $[k_0,j_0]$ on chip
\State accumulate the tile product into the $C$-tile \Comment{each loaded byte reused $T_N$ or $T_M$ times}
\State accumulate the tile product on chip \Comment{reuse $T_N$/$T_M$ per byte}
\EndFor
\State write the $C$-tile back to memory
\EndFor
book/quarto/contents/vol1/model_compression/model_compression.qmd
+4 -2
View File
@@ -4051,14 +4051,16 @@ Walk through the PTQ pipeline in @fig-ptq-calibration step by step. A calibratio
```pseudocode
#| label: algo-ptq-calibration
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Post-training quantization calibration}
\begin{algorithmic}
\Require pretrained model $f_\theta$; representative calibration set $C$; bit width $b$; granularity (layer/group/channel); method (max, entropy, percentile)
\Ensure quantized weights and per-group range metadata $\{(\alpha_g, \beta_g, s_g, z_g)\}$
\State attach observers to weights and selected activation tensors, at the chosen granularity
\For{each batch in $C$}
\State run $f_\theta$ without updating parameters; record per-group value distributions \Comment{calibration pass}
\For{each batch in $C$} \Comment{calibration pass}
\State run $f_\theta$ in inference; record per-group value distributions
\EndFor
\For{each quantization group $g$}
\State select clipping range $[\alpha_g, \beta_g]$ by the chosen method
book/quarto/contents/vol1/nn_computation/nn_computation.qmd
+9 -5
View File
@@ -4236,19 +4236,21 @@ The mathematical foundations of backpropagation provide the theoretical basis fo
```pseudocode
#| label: algo-backpropagation-training-step
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Backpropagation training step}
\begin{algorithmic}
\Require mini-batch inputs $\mathbf{X}$, labels $\mathbf{y}$; $N_L$ layers with parameters $\{\mathbf{W}^{(\ell)}, \mathbf{b}^{(\ell)}\}_{\ell=1}^{N_L}$; loss $\mathcal{L}$
\Ensure parameter gradients $\{\nabla_{\mathbf{W}^{(\ell)}}\mathcal{L}, \nabla_{\mathbf{b}^{(\ell)}}\mathcal{L}\}$ for the optimizer
\State $\mathbf{A}^{(0)} \gets \mathbf{X}$
\For{$\ell = 1$ to $N_L$}
\State compute $\mathbf{Z}^{(\ell)}, \mathbf{A}^{(\ell)}$; save the activation and local values its derivative needs \Comment{forward pass}
\For{$\ell = 1$ to $N_L$} \Comment{forward pass}
\State compute $\mathbf{Z}^{(\ell)}, \mathbf{A}^{(\ell)}$; save the values its derivative needs
\EndFor
\State compute loss $\mathcal{L}(\mathbf{A}^{(N_L)}, \mathbf{y})$; seed the output gradient $\nabla_{\mathbf{A}^{(N_L)}}\mathcal{L}$
\For{$\ell = N_L$ down to $1$}
\For{$\ell = N_L$ down to $1$} \Comment{same local rule repeats}
\State use the saved forward values to compute $\nabla_{\mathbf{W}^{(\ell)}}\mathcal{L}, \nabla_{\mathbf{b}^{(\ell)}}\mathcal{L}$
\State propagate $\nabla_{\mathbf{A}^{(\ell-1)}}\mathcal{L}$ to the previous layer \Comment{same local rule repeats}
\State propagate $\nabla_{\mathbf{A}^{(\ell-1)}}\mathcal{L}$ to the previous layer
\EndFor
\State \Return the accumulated parameter gradients
\end{algorithmic}
@@ -4596,13 +4598,15 @@ class MnistEpochLocal:
The complete training process combines forward propagation, backward propagation, and weight updates into a systematic training loop\index{Training Loop!iterative learning}. This loop repeats until the network achieves satisfactory performance or reaches a predetermined number of iterations.
A single pass through the entire training dataset is called an epoch\index{Epoch!dataset iteration}[^fn-epoch-training-nn]. For MNIST, with `{python} MnistEpochLocal.training_examples_str` training images and a batch size of `{python} MnistEpochLocal.batch_size_str`, each epoch consists of `{python} MnistEpochLocal.epoch_batches_str` batch iterations. @Algo-minibatch-sgd summarizes the complete mini-batch SGD procedure\index{Stochastic Gradient Descent!mini-batch procedure}.
A single pass through the entire training dataset is called an epoch\index{Epoch!dataset iteration}[^fn-epoch-training-nn]. For MNIST, with `{python} MnistEpochLocal.training_examples_str` training images and a batch size of `{python} MnistEpochLocal.batch_size_str`, each epoch consists of `{python} MnistEpochLocal.epoch_batches_str` batch iterations. @Algo-minibatch-sgd lays out the mini-batch SGD\index{Stochastic Gradient Descent!mini-batch procedure} loop: each epoch shuffles the data, steps through it one mini-batch at a time with a forward and backward pass, and applies a single parameter update per batch.
[^fn-epoch-training-nn]: **Epoch**: One complete pass through all training data. The number of epochs is a direct multiplier on total compute cost: a 100-epoch MNIST run executes 100$\times$ more forward and backward passes than a single epoch. At frontier scale, this multiplier becomes the binding constraint: GPT-3 trained for only ~1 epoch over 300 billion tokens because the per-epoch cost already consumed thousands of GPU-weeks. \index{Epoch!training cycles}
```pseudocode
#| label: algo-minibatch-sgd
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Mini-batch stochastic gradient descent}
\begin{algorithmic}
book/quarto/contents/vol1/training/training.qmd
+12 -6
View File
@@ -1148,6 +1148,8 @@ The `optimizer.step()` method is the other boundary: it consumes the current gra
```pseudocode
#| label: algo-adam-update
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Adam parameter update (one optimizer step)}
\begin{algorithmic}
@@ -1155,7 +1157,7 @@ The `optimizer.step()` method is the other boundary: it consumes the current gra
\Ensure updated parameter $\theta_{t+1}$; moment buffers $m_t, v_t$ carried to the next step
\State $m_t \gets \beta_1 m_{t-1} + (1-\beta_1)\, g_t$ \Comment{first moment (momentum)}
\State $v_t \gets \beta_2 v_{t-1} + (1-\beta_2)\, g_t^2$ \Comment{second moment (variance)}
\State $\hat{m}_t \gets m_t / (1-\beta_1^{t})$; \quad $\hat{v}_t \gets v_t / (1-\beta_2^{t})$ \Comment{bias correction}
\State $\hat{m}_t \gets m_t / (1-\beta_1^{t})$; $\hat{v}_t \gets v_t / (1-\beta_2^{t})$ \Comment{bias correction}
\State $\theta_{t+1} \gets \theta_t - \eta\, \hat{m}_t / (\sqrt{\hat{v}_t} + \epsilon)$ \Comment{parameter update}
\end{algorithmic}
\end{algorithm}
@@ -1404,10 +1406,12 @@ To manage these substantial memory requirements, training systems use several so
```pseudocode
#| label: algo-gradient-checkpointing
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Gradient checkpointing (activation recomputation)}
\begin{algorithmic}
\Require $N_L$-layer network; checkpoint set $\mathcal{C} \subseteq \{1,\dots,N_L\}$ (e.g.\ every $\sqrt{N_L}$ layers)
\Require $N_L$-layer network; checkpoint set $\mathcal{C} \subseteq \{1,\dots,N_L\}$ (e.g. every $\sqrt{N_L}$ layers)
\Ensure parameter gradients, at reduced peak activation memory
\For{$\ell = 1$ to $N_L$}
\State forward layer $\ell$; store its activation only if $\ell \in \mathcal{C}$ \Comment{drop the rest}
@@ -4530,21 +4534,23 @@ $$
```pseudocode
#| label: algo-streaming-attention
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{FlashAttention: tiled attention with online softmax}
\begin{algorithmic}
\Require queries $Q$, keys $K$, values $V$ for a length-$S$ sequence; tile size $b$
\Ensure attention output $Y$, without materializing the $S\times S$ score matrix
\For{each query tile $Q_i$}
\State $Y_i^{\text{acc}} \gets \mathbf{0}$; row-wise max $m_i \gets -\infty$; row-wise normalizer $l_i \gets 0$ \Comment{in SRAM}
\State in SRAM: $Y_i^{\text{acc}} \gets \mathbf{0}$, $m_i \gets -\infty$, $l_i \gets 0$
\For{each key/value tile $(K_j, V_j)$}
\State load $Q_i, K_j, V_j$ into SRAM;\ $Z_{ij} \gets Q_i K_j^\top / \sqrt{d_k}$
\State load $Q_i, K_j, V_j$ into SRAM; $Z_{ij} \gets Q_i K_j^\top / \sqrt{d_k}$
\State $m_i' \gets \max(m_i,\operatorname{rowmax}(Z_{ij}))$ \Comment{new stable max}
\State $Y_i^{\text{acc}} \gets Y_i^{\text{acc}}\odot e^{m_i-m_i'} + e^{Z_{ij}-m_i'}V_j$
\State $l_i \gets l_i\odot e^{m_i-m_i'} + \operatorname{rowsum}(e^{Z_{ij}-m_i'})$;\quad $m_i \gets m_i'$
\State $l_i \gets l_i\odot e^{m_i-m_i'} + \operatorname{rowsum}(e^{Z_{ij}-m_i'})$; $m_i \gets m_i'$
\State discard $Z_{ij}$ \Comment{never written to HBM}
\EndFor
\State $Y_i \gets Y_i^{\text{acc}} / l_i$;\ write $Y_i$ to HBM
\State $Y_i \gets Y_i^{\text{acc}} / l_i$; write $Y_i$ to HBM
\EndFor
\end{algorithmic}
\end{algorithm}
book/quarto/contents/vol2/collective_communication/collective_communication.qmd
+2
View File
@@ -896,6 +896,8 @@ The algorithm splits the vector of size $M$ into $N$ chunks and then alternates
```pseudocode
#| label: algo-ring-allreduce
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Ring AllReduce}
\begin{algorithmic}
book/quarto/contents/vol2/distributed_training/distributed_training.qmd
+4 -2
View File
@@ -2137,18 +2137,20 @@ At the schedule level, 1F1B is a warmup, steady-state, and drain loop. @Algo-one
```pseudocode
#| label: algo-one-forward-one-backward-pipeline
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{One-forward-one-backward pipeline schedule}
\begin{algorithmic}
\Require $p$ ordered pipeline stages; $m$ microbatches; activation links $j\to j{+}1$ and gradient links $j{+}1\to j$
\Ensure one optimizer update after gradients accumulate across all $m$ microbatches
\State split the global batch into $m$ microbatches; assign consecutive layers to the $p$ stages
\State \textbf{warm-up:} launch forwards from stage 0, forwarding each activation downstream as produced
\State warm-up: launch forwards from stage 0, forwarding each activation downstream as produced
\For{each stage in steady state}
\State alternate one backward (oldest ready microbatch) with one forward (next microbatch)
\State release a microbatch's activation as soon as that stage's backward consumes it
\EndFor
\State \textbf{drain:} finish the remaining backward passes in reverse stage order
\State drain: finish the remaining backward passes in reverse stage order
\State apply the optimizer update once every stage has accumulated gradients for all $m$ microbatches
\end{algorithmic}
\end{algorithm}
book/quarto/contents/vol2/inference/inference.qmd
+9 -3
View File
@@ -1012,6 +1012,8 @@ Continuous batching (also called iteration-level batching) decouples batch membe
```pseudocode
#| label: algo-continuous-batching-decode-scheduler
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Continuous batching decode scheduler}
\begin{algorithmic}
@@ -2648,6 +2650,8 @@ Speculative decoding has three operational phases: a Draft phase\index{Speculati
```pseudocode
#| label: algo-speculative-decoding
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Speculative decoding}
\begin{algorithmic}
@@ -3971,15 +3975,17 @@ The procedure is deliberately small, but the load signal must match the serving
```pseudocode
#| label: algo-power-two-choices-routing
#| pdf-line-number: true
#| pdf-right-comment: false
#| html-comment-delimiter: "▷"
\begin{algorithm}
\caption{Power-of-two choices request routing}
\begin{algorithmic}
\Require replica set $\mathcal{S}$; request $r$; capacity weights $w_s$; per-replica work signal $W_s$ (queue length, active tokens, predicted work, or KV-cache occupancy)
\Ensure a selected replica for $r$
\State sample two candidate replicas $s_a, s_b$ from $\mathcal{S}$ \Comment{uniform if homogeneous, capacity-weighted if not}
\State sample two candidate replicas $s_a, s_b$ from $\mathcal{S}$ \Comment{capacity-weighted}
\State query $W_{s_a}, W_{s_b}$; estimate incremental work $\hat{c}(r)$ (expected output tokens or KV pages)
\State $u_s \gets (W_s + \hat{c}(r)) / w_s$ for $s \in \{s_a, s_b\}$ \Comment{normalized post-admission load}
\State route $r$ to the candidate with smaller $u_s$ \Comment{ties broken by locality or cache affinity}
\State $u_s \gets (W_s + \hat{c}(r)) / w_s$ for $s \in \{s_a, s_b\}$ \Comment{post-admission load}
\State route $r$ to the candidate with smaller $u_s$ \Comment{ties: cache affinity}
\State update the chosen replica's local admission state
\end{algorithmic}
\end{algorithm}
book/quarto/tex/header-includes.tex
+10
View File
@@ -63,6 +63,16 @@ labelformat=mylabel,justification=raggedright,singlelinecheck=false,font={ninept
% Utility packages
\usepackage{etoolbox} % For patching commands and environments
% Algorithm pseudocode (algpseudocodex): in the narrow text column, forbid
% mid-word hyphenation so algorithm steps wrap at spaces, not inside words.
% Deferred to begin-document and guarded so it applies only when a chapter
% actually loads the pseudocode package.
\AtBeginDocument{%
\ifdef{\algorithmic}{%
\AtBeginEnvironment{algorithmic}{\hyphenpenalty=10000\exhyphenpenalty=10000\tolerance=3000\emergencystretch=2.5em}%
}{}%
}
% Page layout and headers
\usepackage{fancyhdr} % Custom headers and footers
\usepackage{geometry} % Page dimensions and margins