diff --git a/tinytorch/INSTRUCTOR.md b/tinytorch/INSTRUCTOR.md index 8d8d477329..676a91670e 100644 --- a/tinytorch/INSTRUCTOR.md +++ b/tinytorch/INSTRUCTOR.md @@ -180,7 +180,7 @@ def memory_footprint(self): - Shows understanding of memory calculation - Missing proper dtype handling -### Module 05: Autograd - Backward Pass +### Module 06: Autograd - Backward Pass **Excellent Solution (9-10 points)**: ```python diff --git a/tinytorch/milestones/01_1957_perceptron/01_rosenblatt_forward.py b/tinytorch/milestones/01_1957_perceptron/01_rosenblatt_forward.py index 356017db59..e9f10c91e1 100644 --- a/tinytorch/milestones/01_1957_perceptron/01_rosenblatt_forward.py +++ b/tinytorch/milestones/01_1957_perceptron/01_rosenblatt_forward.py @@ -405,9 +405,9 @@ def main(): # Next steps next_steps = ( "[bold]Complete Modules 05-07 to unlock TRAINING:[/bold]\n\n" - " [cyan]•[/cyan] Module 05 (Autograd): Calculate gradients automatically\n" - " [cyan]•[/cyan] Module 06 (Optimizers): Update weights intelligently\n" - " [cyan]•[/cyan] Module 07 (Training): Put it all together\n\n" + " [cyan]•[/cyan] Module 06 (Autograd): Calculate gradients automatically\n" + " [cyan]•[/cyan] Module 07 (Optimizers): Update weights intelligently\n" + " [cyan]•[/cyan] Module 08 (Training): Put it all together\n\n" "[dim]Then return to this SAME perceptron and watch it achieve 95%+!\n" "You'll see random → intelligent through the power of learning![/dim]" ) diff --git a/tinytorch/milestones/02_1969_xor/02_xor_solved.py b/tinytorch/milestones/02_1969_xor/02_xor_solved.py index 5fc3c1a97d..67af73d859 100755 --- a/tinytorch/milestones/02_1969_xor/02_xor_solved.py +++ b/tinytorch/milestones/02_1969_xor/02_xor_solved.py @@ -18,9 +18,9 @@ Watch a multi-layer network SOLVE the "impossible" XOR problem that stumped AI f Module 02 (Activations) : YOUR ReLU and Sigmoid (non-linearity!) Module 03 (Layers) : YOUR Linear layers (multiple layers!) Module 04 (Losses) : YOUR loss function - Module 05 (Autograd) : YOUR backpropagation through hidden layers - Module 06 (Optimizers) : YOUR SGD optimizer - Module 07 (Training) : YOUR training loop + Module 06 (Autograd) : YOUR backpropagation through hidden layers + Module 07 (Optimizers) : YOUR SGD optimizer + Module 08 (Training) : YOUR training loop ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 🏗️ ARCHITECTURE (The Multi-Layer Solution): @@ -111,7 +111,7 @@ console = Console() # │ Module 04: Loss │ BinaryCrossEntropy measures │ Guides learning toward │ # │ │ how wrong predictions are │ correct XOR outputs │ # │ │ │ │ -# │ Module 05: Autograd │ .backward() computes gradients │ Gradients flow through │ +# │ Module 06: Autograd │ .backward() computes gradients │ Gradients flow through │ # │ │ for BOTH layers automatically │ hidden layer to inputs! │ # │ │ │ │ # │ Module 06: SGD │ Updates 13 parameters │ Adjusts weights to minimize │ diff --git a/tinytorch/milestones/05_2017_transformer/01_vaswani_attention.py b/tinytorch/milestones/05_2017_transformer/01_vaswani_attention.py index 72eef92d75..ee9509aad1 100755 --- a/tinytorch/milestones/05_2017_transformer/01_vaswani_attention.py +++ b/tinytorch/milestones/05_2017_transformer/01_vaswani_attention.py @@ -358,7 +358,7 @@ def train_epoch(model, dataloader, optimizer, loss_fn): # Use YOUR DataLoader for batched training! for input_batch, target_batch in dataloader: batch_size = input_batch.shape[0] - + logits = model(input_batch) # Reshape for loss computation @@ -376,7 +376,7 @@ def train_epoch(model, dataloader, optimizer, loss_fn): pred = np.argmax(logits.data, axis=-1) for i in range(batch_size): if np.array_equal(pred[i], target_batch.data[i]): - correct_sequences += 1 + correct_sequences += 1 total_samples += batch_size return total_loss / total_samples, (correct_sequences / total_samples) * 100 diff --git a/tinytorch/milestones/06_2018_mlperf/01_optimization_olympics.py b/tinytorch/milestones/06_2018_mlperf/01_optimization_olympics.py index 6199bc729c..3ab1786ff3 100644 --- a/tinytorch/milestones/06_2018_mlperf/01_optimization_olympics.py +++ b/tinytorch/milestones/06_2018_mlperf/01_optimization_olympics.py @@ -17,7 +17,7 @@ it production-ready. Every technique uses YOUR implementations! ✅ REQUIRED MODULES (Run after Module 19): ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Module 01-03: Tensor, Activations, Layers - YOUR base model - Module 07: Training - YOUR trained model from earlier milestones + Module 08: Training - YOUR trained model from earlier milestones Module 14: Profiling - YOUR Profiler class Module 15: Quantization - YOUR Quantizer class Module 16: Compression - YOUR Compressor class diff --git a/tinytorch/paper/paper.tex b/tinytorch/paper/paper.tex index 4e8cb3c30b..1ad515ef11 100644 --- a/tinytorch/paper/paper.tex +++ b/tinytorch/paper/paper.tex @@ -222,7 +222,7 @@ % Abstract - REVISED: Curriculum design focus \begin{abstract} -Machine learning systems engineering requires understanding framework internals: why optimizers consume memory, when computational complexity becomes prohibitive, how to navigate accuracy-latency-memory tradeoffs. Yet current ML education separates algorithms from systems—students learn gradient descent without measuring memory, attention mechanisms without profiling costs, training without understanding optimizer overhead. This divide leaves graduates unable to debug production failures or make informed engineering decisions. We present TinyTorch, a build-from-scratch curriculum where students implement PyTorch's core components (tensors, autograd, optimizers, neural networks) to gain framework transparency. Three pedagogical patterns address the gap: \textbf{progressive disclosure} gradually reveals complexity (gradient features exist from Module 01, activate in Module 05); \textbf{systems-first curriculum} embeds memory profiling from the start; \textbf{historical milestone validation} recreates nearly 70 years of ML breakthroughs (1958--2025) using exclusively student-implemented code. These patterns are grounded in learning theory (situated cognition, cognitive load theory) but represent testable hypotheses requiring empirical validation. The 20-module curriculum provides complete open-source infrastructure at \texttt{mlsysbook.ai/tinytorch}. +Machine learning systems engineering requires understanding framework internals: why optimizers consume memory, when computational complexity becomes prohibitive, how to navigate accuracy-latency-memory tradeoffs. Yet current ML education separates algorithms from systems—students learn gradient descent without measuring memory, attention mechanisms without profiling costs, training without understanding optimizer overhead. This divide leaves graduates unable to debug production failures or make informed engineering decisions. We present TinyTorch, a build-from-scratch curriculum where students implement PyTorch's core components (tensors, autograd, optimizers, neural networks) to gain framework transparency. Three pedagogical patterns address the gap: \textbf{progressive disclosure} gradually reveals complexity (gradient features exist from Module 01, activate in Module 06); \textbf{systems-first curriculum} embeds memory profiling from the start; \textbf{historical milestone validation} recreates nearly 70 years of ML breakthroughs (1958--2025) using exclusively student-implemented code. These patterns are grounded in learning theory (situated cognition, cognitive load theory) but represent testable hypotheses requiring empirical validation. The 20-module curriculum provides complete open-source infrastructure at \texttt{mlsysbook.ai/tinytorch}. \end{abstract} @@ -345,7 +345,7 @@ for batch in DataLoader(data): \end{figure*} -Building systems knowledge alongside ML fundamentals presents three pedagogical challenges: teaching systems thinking early without overwhelming beginners (\Cref{sec:systems}), managing cognitive load when teaching both algorithms and implementation (\Cref{sec:progressive}), and validating student understanding through concrete milestones (\Cref{subsec:milestones}). TinyTorch addresses these through curriculum design inspired by compiler courses~\citep{aho2006compilers}---students build a complete system incrementally, with each module adding functionality while maintaining a working implementation. \Cref{fig:module-flow} illustrates this progression: tensors (Module 01) enable activations (02) and layers (03), which feed into autograd (05), powering optimizers (06) and training (07). Each completed module becomes immediately usable: after Module 03, students build neural networks; after Module 05, automatic differentiation enables training; after Module 13, transformers support language modeling. This structure enables students to construct mental models gradually while seeing immediate results. +Building systems knowledge alongside ML fundamentals presents three pedagogical challenges: teaching systems thinking early without overwhelming beginners (\Cref{sec:systems}), managing cognitive load when teaching both algorithms and implementation (\Cref{sec:progressive}), and validating student understanding through concrete milestones (\Cref{subsec:milestones}). TinyTorch addresses these through curriculum design inspired by compiler courses~\citep{aho2006compilers}---students build a complete system incrementally, with each module adding functionality while maintaining a working implementation. \Cref{fig:module-flow} illustrates this progression: tensors (Module 01) enable activations (02) and layers (03), which feed into dataloader (05) and autograd (06), powering optimizers (07) and training (08). Each completed module becomes immediately usable: after Module 03, students build neural networks; after Module 06, automatic differentiation enables training; after Module 13, transformers support language modeling. This structure enables students to construct mental models gradually while seeing immediate results. TinyTorch serves students transitioning from framework \emph{users} to framework \emph{engineers}: those who have completed introductory ML courses (e.g., CS229, fast.ai) and want to understand PyTorch internals, those planning ML systems research or infrastructure careers, or practitioners debugging production deployment issues. The curriculum assumes NumPy proficiency and basic neural network familiarity but teaches framework architecture from first principles. Students needing immediate GPU/distributed training skills are better served by PyTorch tutorials; those preferring project-based application building will find high-level frameworks more appropriate. The 20-module structure supports flexible pacing: intensive completion, semester integration (parallel with lectures), or self-paced professional development. @@ -354,7 +354,7 @@ This paper makes three contributions, each addressing the systems imperative the \begin{enumerate}[leftmargin=*, itemsep=0.5em] \item \textbf{Build-to-Validate Curriculum} (\Cref{sec:curriculum}): A 20-module learning path where students validate their implementations by recreating historical ML milestones---from Rosenblatt's Perceptron (1958) to modern transformers---using exclusively their own code. This provides concrete correctness criteria and grounds abstract concepts in tangible achievements. -\item \textbf{Progressive Disclosure of Complexity} (\Cref{sec:progressive}): A scaffolding technique that reveals \texttt{Tensor} internals gradually while maintaining a unified mental model. Gradient tracking infrastructure exists from Module 01 but activates only in Module 05, preventing premature cognitive overload while enabling seamless backpropagation later. +\item \textbf{Progressive Disclosure of Complexity} (\Cref{sec:progressive}): A scaffolding technique that reveals \texttt{Tensor} internals gradually while maintaining a unified mental model. Gradient tracking infrastructure exists from Module 01 but activates only in Module 06, preventing premature cognitive overload while enabling seamless backpropagation later. \item \textbf{Systems from Day One} (\Cref{sec:systems}): Memory profiling, computational complexity, and performance analysis are embedded starting in Module 01---not deferred to advanced topics. Students discover that Adam requires $2\times$ optimizer state memory by implementing it, not by reading documentation. \end{enumerate} @@ -553,11 +553,11 @@ class Tensor: return Tensor(self.data @ other.data) \end{lstlisting} -\textbf{Tier 1: Foundation (Modules 01--07).} -Students build the mathematical core enabling neural networks to learn. Systems thinking begins immediately: Module 01 introduces \texttt{memory\_footprint()} before matrix multiplication (\Cref{lst:tensor-memory}), making memory a first-class concept. The tier progresses through tensors, activations, layers, and losses to automatic differentiation (Module 05), where dormant gradient features activate through progressive disclosure (\Cref{sec:progressive}). Students implement optimizers (Module 06), discovering Adam's memory trade-offs through direct measurement (\Cref{sec:systems}). The training loop (Module 07) integrates all components. By tier completion, students recreate three historical milestones: \citet{rosenblatt1958perceptron}'s Perceptron, Minsky and Papert's XOR solution, and \citet{rumelhart1986learning}'s backpropagation targeting 95\%+ on MNIST. Students calculate memory before operations: ``Matrix multiplication A @ B where both are (1000, 1000) FP32 requires 12MB peak memory: 4MB for A, 4MB for B, and 4MB for the output.'' This reasoning becomes automatic. +\textbf{Tier 1: Foundation (Modules 01--08).} +Students build the mathematical core enabling neural networks to learn. Systems thinking begins immediately: Module 01 introduces \texttt{memory\_footprint()} before matrix multiplication (\Cref{lst:tensor-memory}), making memory a first-class concept. The tier progresses through tensors, activations, layers, and losses to data loading (Module 05), then automatic differentiation (Module 06), where dormant gradient features activate through progressive disclosure (\Cref{sec:progressive}). Students implement optimizers (Module 07), discovering Adam's memory trade-offs through direct measurement (\Cref{sec:systems}). The training loop (Module 08) integrates all components. By tier completion, students recreate three historical milestones: \citet{rosenblatt1958perceptron}'s Perceptron, Minsky and Papert's XOR solution, and \citet{rumelhart1986learning}'s backpropagation targeting 95\%+ on MNIST. Students calculate memory before operations: ``Matrix multiplication A @ B where both are (1000, 1000) FP32 requires 12MB peak memory: 4MB for A, 4MB for B, and 4MB for the output.'' This reasoning becomes automatic. -\textbf{Tier 2: Architectures (Modules 08--13).} -Students apply foundation knowledge to modern architectures for vision and language. Module 08 introduces the Dataset abstraction pattern (implementing \texttt{\_\_len\_\_} and \texttt{\_\_getitem\_\_} protocols) and DataLoader with batch collation, teaching how PyTorch's data pipeline transforms individual samples into batched tensors through the iterator protocol. While Module 07 implements basic training loops with manual batching (simple iteration over pre-batched arrays), Module 08 refactors this into production-quality data loading, a pedagogical pattern of ``make it work, then make it right.'' Students first understand training mechanics (forward pass, loss, backward, update), then learn proper data pipeline engineering. TinyTorch ships with two custom educational datasets that install with the repository: \textbf{TinyDigits} (5,000 grayscale handwritten digits, curated from public digit datasets) and \textbf{TinyTalks} (3,000 synthetically-generated conversational Q\&A pairs). These datasets are deliberately small and offline-first: they require no network connectivity during training, consume minimal storage ($<$50MB combined), and train in minutes on CPU-only hardware. This design ensures accessibility for students in regions with limited internet infrastructure, institutional computer labs with restricted network access, and developing countries where cloud-based datasets create barriers to ML education. +\textbf{Tier 2: Architectures (Modules 09--13).} +Students apply foundation knowledge to modern architectures for vision and language. The DataLoader patterns from Module 05 and training infrastructure from Module 08 combine to enable efficient model development. TinyTorch ships with two custom educational datasets that install with the repository: \textbf{TinyDigits} (5,000 grayscale handwritten digits, curated from public digit datasets) and \textbf{TinyTalks} (3,000 synthetically-generated conversational Q\&A pairs). These datasets are deliberately small and offline-first: they require no network connectivity during training, consume minimal storage ($<$50MB combined), and train in minutes on CPU-only hardware. This design ensures accessibility for students in regions with limited internet infrastructure, institutional computer labs with restricted network access, and developing countries where cloud-based datasets create barriers to ML education. The tier then branches into two paths. \textbf{Vision} implements Conv2d with seven explicit nested loops making $O(C_{out} \times H \times W \times C_{in} \times K^2)$ complexity visible before optimization. Students discover weight sharing's dramatic efficiency through direct comparison: Conv2d(3$\rightarrow$32, kernel=3) requires 896 parameters while an equivalent dense layer needs 98,336 parameters (3072 input features $\times$ 32 outputs + 32 bias terms), a 109$\times$ reduction demonstrating how inductive biases enable CNNs to learn spatial patterns without brute-force parameterization. Students validate their implementations by training CNNs targeting 65--75\% CIFAR-10 accuracy~\citep{krizhevsky2009cifar,lecun1998gradient}. @@ -574,13 +574,13 @@ The capstone builds submission infrastructure enabling ML competitions and repro Each module follows a consistent \textbf{Build $\rightarrow$ Use $\rightarrow$ Reflect} pedagogical cycle that integrates implementation, application, and systems reasoning. This structure addresses multiple learning objectives: students construct working components (Build), validate integration with prior modules (Use), and develop systems thinking~\citep{meadows2008thinking} through analysis (Reflect). -\noindent\textbf{Build: Implementation with Explicit Dependencies.} Students implement components in Jupyter notebooks (\texttt{*.py}) with scaffolded guidance. Each module begins with a \emph{connection map}: a visual diagram showing which prior modules students must have completed (prerequisites), what the current module teaches (focus), and what capabilities become available after completion (unlocks). For example, Module 09 (Convolutions) shows prerequisites of Modules 01--07, focus on spatial operations, and unlocks CNN architectures for image classification. These maps make dependency relationships explicit, helping students understand where each module fits in the larger framework architecture. To maintain consistency across all 20 modules, Module 08 (DataLoader) serves as the canonical reference implementation that all modules follow, reducing maintenance burden and enabling consistent community contribution. +\noindent\textbf{Build: Implementation with Explicit Dependencies.} Students implement components in Jupyter notebooks (\texttt{*.py}) with scaffolded guidance. Each module begins with a \emph{connection map}: a visual diagram showing which prior modules students must have completed (prerequisites), what the current module teaches (focus), and what capabilities become available after completion (unlocks). For example, Module 09 (Convolutions) shows prerequisites of Modules 01--08, focus on spatial operations, and unlocks CNN architectures for image classification. These maps make dependency relationships explicit, helping students understand where each module fits in the larger framework architecture. To maintain consistency across all 20 modules, Module 05 (DataLoader) serves as the canonical reference implementation that all modules follow, reducing maintenance burden and enabling consistent community contribution. -\noindent\textbf{Use: Integration Testing Beyond Unit Tests.} Assessment validates both isolated correctness and cross-module integration. Unit tests verify individual component behavior (``Does \texttt{Tensor.reshape()} produce correct output?''), while integration tests validate that components compose into working systems (``Can Module 05 Autograd compute gradients through Module 03 Linear layers?''). Integration tests are critical for TinyTorch's pedagogical model because students may pass Module 03 unit tests but fail when autograd activates in Module 05: their layer implementation doesn't properly propagate \texttt{requires\_grad} through operations or construct computational graphs correctly. +\noindent\textbf{Use: Integration Testing Beyond Unit Tests.} Assessment validates both isolated correctness and cross-module integration. Unit tests verify individual component behavior (``Does \texttt{Tensor.reshape()} produce correct output?''), while integration tests validate that components compose into working systems (``Can Module 06 Autograd compute gradients through Module 03 Linear layers?''). Integration tests are critical for TinyTorch's pedagogical model because students may pass Module 03 unit tests but fail when autograd activates in Module 06: their layer implementation doesn't properly propagate \texttt{requires\_grad} through operations or construct computational graphs correctly. A common failure pattern illustrates this: students implement \texttt{Linear.forward()} that passes unit tests (correct output values), but gradients don't flow during backpropagation because they used NumPy operations directly instead of \texttt{Tensor} operations. When \texttt{x.requires\_grad=True} flows into their layer, the computational graph breaks. Students encounter errors like ``\texttt{AttributeError: 'numpy.ndarray' object has no attribute 'backward'}'' and must debug interface contracts: operations must preserve \texttt{Tensor} types to maintain gradient connectivity. This teaches \emph{interface design}: components must satisfy contracts enabling composition, not just produce correct outputs in isolation. -Module 09 (Convolutions) integration exemplifies this: convolution must work with Module 05's autograd (gradient flow through kernels), Module 06's optimizers (parameter updates), and Module 07's training loop (forward-backward cycles) simultaneously. Students discover that ``passing unit tests'' $\neq$ ``works in the system'' when their Conv2d produces correct outputs but crashes during \texttt{loss.backward()} because they forgot to track intermediate activations for gradient computation. This debugging mirrors professional ML engineering: isolated correctness is insufficient; system integration reveals interface failures. +Module 09 (Convolutions) integration exemplifies this: convolution must work with Module 06's autograd (gradient flow through kernels), Module 07's optimizers (parameter updates), and Module 08's training loop (forward-backward cycles) simultaneously. Students discover that ``passing unit tests'' $\neq$ ``works in the system'' when their Conv2d produces correct outputs but crashes during \texttt{loss.backward()} because they forgot to track intermediate activations for gradient computation. This debugging mirrors professional ML engineering: isolated correctness is insufficient; system integration reveals interface failures. \noindent\textbf{Reflect: Systems Analysis Questions.} Each module concludes with systems reasoning prompts measuring conceptual understanding beyond syntactic correctness. Memory analysis questions ask students to calculate footprints (``A (256, 256) Conv2d layer with 64 input and 128 output channels requires how much memory?''). Complexity analysis prompts probe asymptotic understanding (``Why is attention $O(N^2)$? Demonstrate by doubling sequence length and measuring memory growth.''). Design trade-off questions assess engineering judgment (``Adam requires 2$\times$ optimizer state memory (momentum and variance) but converges faster than SGD. When is the 4$\times$ total training memory trade-off worth it?''). These open-ended questions assess transfer~\citep{perkins1992transfer}: can students apply learned concepts to novel scenarios not seen in exercises? @@ -599,9 +599,9 @@ Second, \textbf{implementation validation beyond unit tests}: Milestones address \begin{enumerate} \item \textbf{1958 Perceptron} (after Module 04): Train Rosenblatt's original single-layer perceptron on linearly separable classification. Students import \texttt{from tinytorch.core import Tensor; from tinytorch.nn import Linear, Sigmoid}, their framework now supports single-layer networks. -\item \textbf{1969 XOR Solution} (after Module 07): Solve Minsky's ``impossible'' XOR problem with multi-layer perceptrons, proving critics wrong. Validates that autograd enables non-linear learning. +\item \textbf{1969 XOR Solution} (after Module 08): Solve Minsky's ``impossible'' XOR problem with multi-layer perceptrons, proving critics wrong. Validates that autograd enables non-linear learning. -\item \textbf{1986 MLP Revival} (after Module 07): Handwritten digit recognition demonstrating backpropagation's power. Requires Modules 01--07 working together (tensor operations, activations, layers, losses, autograd, optimizers, training). Students import \texttt{from tinytorch.optim import SGD; from tinytorch.nn import CrossEntropyLoss}, their framework trains multi-layer networks end-to-end on MNIST digits. +\item \textbf{1986 MLP Revival} (after Module 08): Handwritten digit recognition demonstrating backpropagation's power. Requires Modules 01--08 working together (tensor operations, activations, layers, losses, dataloader, autograd, optimizers, training). Students import \texttt{from tinytorch.optim import SGD; from tinytorch.nn import CrossEntropyLoss}, their framework trains multi-layer networks end-to-end on MNIST digits. \item \textbf{1998 CNN Revolution} (after Module 09): Image classification demonstrating convolutional architectures' advantage~\citep{krizhevsky2009cifar,lecun1998gradient}. Students import \texttt{from tinytorch.nn import Conv2d, MaxPool2d}, training both MLP and CNN on CIFAR-10 to measure architectural improvements themselves through direct comparison. @@ -633,9 +633,9 @@ This section details how TinyTorch implements progressive disclosure: a pattern \subsection{Pattern Implementation} -TinyTorch's \texttt{Tensor} class includes gradient-related attributes from Module 01, but they remain dormant until Module 05 activates them through monkey-patching (\Cref{lst:dormant-tensor,lst:activation}). \Cref{fig:progressive-timeline} visualizes this activation timeline across the curriculum. +TinyTorch's \texttt{Tensor} class includes gradient-related attributes from Module 01, but they remain dormant until Module 06 activates them through monkey-patching (\Cref{lst:dormant-tensor,lst:activation}). \Cref{fig:progressive-timeline} visualizes this activation timeline across the curriculum. -\begin{lstlisting}[caption={\textbf{Dormant Gradient Infrastructure.} Module 01 Tensor includes \texttt{.backward()}, \texttt{.grad}, and \texttt{.requires\_grad}---visible but inactive until Module 05 activation.},label=lst:dormant-tensor,float=t] +\begin{lstlisting}[caption={\textbf{Dormant Gradient Infrastructure.} Module 01 Tensor includes \texttt{.backward()}, \texttt{.grad}, and \texttt{.requires\_grad}---visible but inactive until Module 06 activation.},label=lst:dormant-tensor,float=t] # Module 01: Foundation Tensor class Tensor: def __init__(self, data, requires_grad=False): @@ -647,14 +647,14 @@ class Tensor: self._backward = None def backward(self, gradient=None): - """No-op until Module 05""" + """No-op until Module 06""" pass def __mul__(self, other): return Tensor(self.data * other.data) \end{lstlisting} -\begin{lstlisting}[caption={\textbf{Autograd Activation.} Module 05 monkey-patches Tensor to enable gradient computation.},label=lst:activation,float=t] +\begin{lstlisting}[caption={\textbf{Autograd Activation.} Module 06 monkey-patches Tensor to enable gradient computation.},label=lst:activation,float=t] def enable_autograd(): """Monkey-patch Tensor with gradients""" def backward(self, gradient=None): @@ -671,7 +671,7 @@ def enable_autograd(): Tensor.backward = backward print("Autograd activated!") -# Module 05 usage +# Module 06 usage enable_autograd() x = Tensor([3.0], requires_grad=True) y = x * x # y = 9.0 @@ -697,7 +697,7 @@ print(x.grad) # [6.0] - dy/dx = 2x \node[below, font=\tiny] at (\x, -0.3) {\texttt{M\label}}; } -% Module 05 activation boundary - thicker and highlighted +% Module 06 activation boundary - thicker and highlighted \draw[red!60, very thick] (6, 0) -- (6, 5.5); \node[above, font=\scriptsize\bfseries, red!70] at (6, 5.7) {ACTIVATE}; @@ -706,7 +706,7 @@ print(x.grad) # [6.0] - dy/dx = 2x \node[active, minimum width=4.0cm, anchor=center] at (3.5, 1.0) {\texttt{.data}, \texttt{.shape}}; \node[left, font=\tiny] at (0.2, 1.0) {Core}; -% Layer 2: Gradient features - boxes meet exactly at x=6 (Module 05 line) +% Layer 2: Gradient features - boxes meet exactly at x=6 (Module 06 line) % .requires_grad \node[dormant] at (6, 2.2) {\texttt{.requires\_grad}}; \node[active] at (6, 2.2) {\texttt{.requires\_grad}}; @@ -728,7 +728,7 @@ print(x.grad) # [6.0] - dy/dx = 2x }; \node[align=center, font=\tiny, text width=4.5cm] at (10, 6.5) { - \textbf{Modules 05--20:}\\ + \textbf{Modules 06--20:}\\ Autograd fully active\\ Gradients flow automatically }; @@ -738,7 +738,7 @@ print(x.grad) # [6.0] - dy/dx = 2x \node[active, minimum width=1.0cm, minimum height=0.4cm, anchor=center] at (5.5, -1.2) {Active}; \end{tikzpicture} -\caption{\textbf{Progressive Disclosure.} Runtime feature activation manages cognitive load. From Module 01, students see the complete Tensor API including gradient methods (\texttt{.backward()}, \texttt{.grad}, \texttt{.requires\_grad}), but these features remain dormant (gray, dashed). In Module 05, runtime enhancement activates full autograd functionality (orange, solid) without breaking earlier code. Three learning benefits: (1) students learn the complete API early, avoiding interface surprise later; (2) Module 01 code continues working unchanged when autograd activates (forward compatibility); (3) visible but inactive features create curiosity-driven questions motivating curriculum progression.} +\caption{\textbf{Progressive Disclosure.} Runtime feature activation manages cognitive load. From Module 01, students see the complete Tensor API including gradient methods (\texttt{.backward()}, \texttt{.grad}, \texttt{.requires\_grad}), but these features remain dormant (gray, dashed). In Module 06, runtime enhancement activates full autograd functionality (orange, solid) without breaking earlier code. Three learning benefits: (1) students learn the complete API early, avoiding interface surprise later; (2) Module 01 code continues working unchanged when autograd activates (forward compatibility); (3) visible but inactive features create curiosity-driven questions motivating curriculum progression.} \label{fig:progressive-timeline} \end{figure*} @@ -746,7 +746,7 @@ print(x.grad) # [6.0] - dy/dx = 2x Progressive disclosure is grounded in cognitive load theory~\citep{sweller1988cognitive} and threshold concept pedagogy~\citep{meyer2003threshold}. The cognitive load hypothesis (early API familiarity reduces future load when features activate) competes with potential split-attention effects from visible but dormant features. Autograd represents a threshold concept---transformative and troublesome---made visible early (dormant) but activatable when students are cognitively ready. Empirical measurement planned for Fall 2025 (\Cref{sec:future-work}) will quantify the net cognitive load impact. -\noindent\textbf{Implementation Choice: Monkey-Patching vs. Inheritance.} Alternative designs include inheritance (\texttt{TensorV1}/\texttt{TensorV2}) or composition. We chose monkey-patching because it mirrors PyTorch 0.4's Variable-Tensor merger via runtime consolidation. The software engineering trade-off (global state modification) is explicitly discussed in Module 05's reflection questions. +\noindent\textbf{Implementation Choice: Monkey-Patching vs. Inheritance.} Alternative designs include inheritance (\texttt{TensorV1}/\texttt{TensorV2}) or composition. We chose monkey-patching because it mirrors PyTorch 0.4's Variable-Tensor merger via runtime consolidation. The software engineering trade-off (global state modification) is explicitly discussed in Module 06's reflection questions. \subsection{Production Framework Alignment} @@ -844,7 +844,7 @@ The Optimization Tier completes the systems-first integration arc: students who Translating curriculum design into effective classroom practice requires addressing integration models, infrastructure accessibility, and student support structures. This section presents deployment patterns validated through pilot implementations and institutional feedback. \noindent\textbf{Textbook Integration.} -TinyTorch serves as the hands-on implementation companion to the \emph{Machine Learning Systems} textbook~\citep{mlsysbook2025} (\texttt{mlsysbook.ai}), creating synergy between theoretical foundations and systems engineering practice. While the textbook covers the full ML lifecycle—data engineering, training architectures, deployment monitoring, robust operations, and sustainable AI—TinyTorch provides the complementary experience of building core infrastructure from first principles. This integration enables a complete educational pathway: students study production ML systems architecture in the textbook (Chapter 4: distributed training patterns, Chapter 7: quantization strategies), then implement those same abstractions in TinyTorch (Module 05: autograd for backpropagation, Module 15: INT8 quantization). The two resources address different aspects of the same educational gap: understanding both \emph{how production systems work} (textbook's systems architecture perspective) and \emph{how to build them yourself} (TinyTorch's implementation depth). +TinyTorch serves as the hands-on implementation companion to the \emph{Machine Learning Systems} textbook~\citep{mlsysbook2025} (\texttt{mlsysbook.ai}), creating synergy between theoretical foundations and systems engineering practice. While the textbook covers the full ML lifecycle—data engineering, training architectures, deployment monitoring, robust operations, and sustainable AI—TinyTorch provides the complementary experience of building core infrastructure from first principles. This integration enables a complete educational pathway: students study production ML systems architecture in the textbook (Chapter 4: distributed training patterns, Chapter 7: quantization strategies), then implement those same abstractions in TinyTorch (Module 06: autograd for backpropagation, Module 15: INT8 quantization). The two resources address different aspects of the same educational gap: understanding both \emph{how production systems work} (textbook's systems architecture perspective) and \emph{how to build them yourself} (TinyTorch's implementation depth). \subsection{Integration Models} \label{subsec:integration} @@ -870,7 +870,7 @@ TinyTorch's three-tier architecture (Foundation, Architecture, Optimization) ena \textbf{Configuration 3: Optimization Focus (Modules 14--19 only).} Students import pre-built \texttt{tinytorch.nn} and \texttt{tinytorch.optim} packages from Configurations 1--2, implementing only production optimization techniques: profiling, quantization, compression, memoization, acceleration, and benchmarking. This configuration targets production ML courses, TinyML workshops, or edge deployment seminars where students already understand framework basics but need systems optimization depth. Students complete Milestone 6 (MLPerf Benchmarks capstone) demonstrating reproducible benchmarking and optimization workflows. Upon completion, students optimize existing models for deployment constraints. Addresses key pedagogical limitation: students interested in quantization shouldn't need to re-implement autograd first. -These configurations support "build what you're learning, import what you need" pedagogy. Configuration 3 students focus on optimization while treating Foundation/Architecture as trusted dependencies, mirroring professional practice where engineers specialize rather than rebuilding entire stacks. The three-tier structure also enables multi-semester deployments aligned with academic terms, and hybrid integration where TinyTorch modules augment PyTorch-first courses by revealing framework internals (e.g., implementing Module 05 autograd to understand \texttt{loss.backward()}, or Module 09 convolution to demystify \texttt{torch.nn.Conv2d}). +These configurations support "build what you're learning, import what you need" pedagogy. Configuration 3 students focus on optimization while treating Foundation/Architecture as trusted dependencies, mirroring professional practice where engineers specialize rather than rebuilding entire stacks. The three-tier structure also enables multi-semester deployments aligned with academic terms, and hybrid integration where TinyTorch modules augment PyTorch-first courses by revealing framework internals (e.g., implementing Module 06 autograd to understand \texttt{loss.backward()}, or Module 09 convolution to demystify \texttt{torch.nn.Conv2d}). \subsection{ML Systems Competency Coverage} \label{subsec:coverage} @@ -953,13 +953,13 @@ def memory_footprint(self): This scaffolding~\citep{vygotsky1978mind} makes educational objectives explicit while enabling automated grading. The \texttt{name} field identifies the exercise, \texttt{points} assigns weight, and the description provides context before students see code cells. -\textbf{Handling Autograder Edge Cases}: Pure Python convolution (Module 09) may exceed default 30-second timeout on slower hardware; we set 5-minute timeouts and provide vectorized reference solutions for comparison. Critical modules (05 Autograd, 09 CNNs) include manual review of 20\% of submissions to catch conceptual errors missed by unit tests. All modules include \texttt{assert numpy.\_\_version\_\_ >= '1.20'} dependency validation. +\textbf{Handling Autograder Edge Cases}: Pure Python convolution (Module 09) may exceed default 30-second timeout on slower hardware; we set 5-minute timeouts and provide vectorized reference solutions for comparison. Critical modules (06 Autograd, 09 CNNs) include manual review of 20\% of submissions to catch conceptual errors missed by unit tests. All modules include \texttt{assert numpy.\_\_version\_\_ >= '1.20'} dependency validation. \textbf{Projected Scalability}: Small courses (30 students) can grade in approximately 10 minutes per module on instructor laptop, medium courses (100 students) require approximately 30 minutes on dedicated grading server, while MOOCs (1000+ students) can achieve 2-hour turnaround via parallelized cloud autograding. These projections assume average grading time of 45 seconds per module submission on 4-core systems. Full-scale deployment validation planned for Fall 2025 (\Cref{sec:discussion}). \subsection{Automated Assessment Infrastructure} -TinyTorch integrates NBGrader~\citep{blank2019nbgrader} for scalable automated assessment~\citep{thompson2008bloom}. Each module contains \textbf{solution cells} (scaffolded implementations with grade metadata), \textbf{test cells} (locked autograded tests preventing modification), and \textbf{point allocations} reflecting pedagogical priorities (Module 05 Autograd: 100 points; Module 01 Tensor: 60 points). Students validate correctness locally before submission, enabling immediate feedback. +TinyTorch integrates NBGrader~\citep{blank2019nbgrader} for scalable automated assessment~\citep{thompson2008bloom}. Each module contains \textbf{solution cells} (scaffolded implementations with grade metadata), \textbf{test cells} (locked autograded tests preventing modification), and \textbf{point allocations} reflecting pedagogical priorities (Module 06 Autograd: 100 points; Module 01 Tensor: 60 points). Students validate correctness locally before submission, enabling immediate feedback. This infrastructure enables deployment in MOOCs and large classrooms where manual grading proves infeasible. Instructors configure NBGrader to collect submissions, execute tests in sandboxed environments, and generate grade reports automatically. @@ -968,7 +968,7 @@ This infrastructure enables deployment in MOOCs and large classrooms where manua Unlike tutorial-style notebooks creating isolated code, TinyTorch modules export to a package structure inspired by PyTorch's API organization. Critically, each completed module becomes immediately usable: students build a working framework progressively, not isolated exercises. Module 01 exports to \texttt{tinytorch.core.tensor}, Module 09 to \texttt{tinytorch.nn.conv}, enabling import patterns familiar to PyTorch users that grow with each module completed. -As students complete modules, their framework accumulates capabilities. After Module 03, students can import and use layers; after Module 05, autograd enables training; after Module 09, CNNs become available. This progressive accumulation creates tangible evidence of progress: students see their framework grow from basic tensors to a complete ML system. \Cref{lst:progressive-imports} illustrates how imports expand as modules are completed: +As students complete modules, their framework accumulates capabilities. After Module 03, students can import and use layers; after Module 06, autograd enables training; after Module 09, CNNs become available. This progressive accumulation creates tangible evidence of progress: students see their framework grow from basic tensors to a complete ML system. \Cref{lst:progressive-imports} illustrates how imports expand as modules are completed: \begin{lstlisting}[caption={\textbf{Progressive Imports.} Framework capabilities grow module-by-module as students complete implementations.},label=lst:progressive-imports,float=t] # After Module 01: Basic tensors @@ -1002,7 +1002,7 @@ Effective deployment requires structured TA support beyond instructor guidance. \textbf{TA Preparation}: TAs should develop deep familiarity with critical modules where students commonly struggle—Modules 05 (Autograd), 09 (CNNs), and 13 (Transformers)—by completing these modules themselves and intentionally introducing bugs to understand common error patterns. The repository provides \texttt{TA\_GUIDE.md} documenting frequent student errors (gradient shape mismatches, disconnected computational graphs, broadcasting failures) and debugging strategies. -\textbf{Office Hour Demand Patterns}: Student help requests are expected to cluster around conceptually challenging modules, with autograd (Module 05) likely generating higher office hour demand than foundation modules. Instructors should anticipate demand spikes by scheduling additional TA capacity during critical modules, providing pre-recorded debugging walkthroughs, and establishing async support channels (discussion forums with guaranteed response times). +\textbf{Office Hour Demand Patterns}: Student help requests are expected to cluster around conceptually challenging modules, with autograd (Module 06) likely generating higher office hour demand than foundation modules. Instructors should anticipate demand spikes by scheduling additional TA capacity during critical modules, providing pre-recorded debugging walkthroughs, and establishing async support channels (discussion forums with guaranteed response times). \textbf{Grading Infrastructure}: While NBGrader automates 70-80\% of assessment, critical modules benefit from manual review of implementation quality and conceptual understanding. TAs should focus manual grading on: (1) code clarity and design choices, (2) edge case handling, (3) computational complexity analysis, and (4) memory profiling insights. Sample solutions and grading rubrics in \texttt{INSTRUCTOR.md} calibrate evaluation standards. @@ -1047,7 +1047,7 @@ TinyTorch's current implementation contains gaps requiring future work. \textbf{Language and accessibility}: Materials exist exclusively in English. Modular structure facilitates translation; community contributions welcome. Code examples omit type annotations (e.g., \texttt{def forward(self, x: Tensor) -> Tensor:}) to reduce visual complexity for students learning ML concepts simultaneously. While this prioritizes pedagogical clarity, it means students don't practice type-driven development increasingly standard in production ML codebases. Future iterations could introduce type hints progressively: omitting them in early modules (01--05), then adding them in optimization modules (14--18) where interface contracts become critical. -\textbf{Forward dependency prevention}: A recurring curriculum maintenance challenge is forward dependency creep---advanced concepts leaking into foundational modules through helper functions, error messages, or test cases that assume knowledge students haven't yet acquired. For example, an error message in Module 03 (Layers) that mentions ``computational graph'' assumes Module 05 (Autograd) knowledge. Maintaining pedagogical ordering requires vigilance during curriculum updates; future work could automate this validation through CI/CD checks that flag cross-module dependencies violating prerequisite ordering. +\textbf{Forward dependency prevention}: A recurring curriculum maintenance challenge is forward dependency creep---advanced concepts leaking into foundational modules through helper functions, error messages, or test cases that assume knowledge students haven't yet acquired. For example, an error message in Module 03 (Layers) that mentions ``computational graph'' assumes Module 06 (Autograd) knowledge. Maintaining pedagogical ordering requires vigilance during curriculum updates; future work could automate this validation through CI/CD checks that flag cross-module dependencies violating prerequisite ordering. \section{Future Directions} \label{sec:future-work} diff --git a/tinytorch/site/_static/demos/tapes/03-milestone-unlocked.tape b/tinytorch/site/_static/demos/tapes/03-milestone-unlocked.tape index 6328963d89..a1ee45bd50 100644 --- a/tinytorch/site/_static/demos/tapes/03-milestone-unlocked.tape +++ b/tinytorch/site/_static/demos/tapes/03-milestone-unlocked.tape @@ -93,7 +93,7 @@ Enter Sleep 4s # Let viewers see modules 01-06 are completed # ============================================================================== -# STEP 2: UNLOCK MILESTONE - Complete module 07 (Training) +# STEP 2: UNLOCK MILESTONE - Complete module 08 (Training) # ============================================================================== Type "tito module complete 07" diff --git a/tinytorch/site/credits.md b/tinytorch/site/credits.md index d3b2c1e93f..121d33a32f 100644 --- a/tinytorch/site/credits.md +++ b/tinytorch/site/credits.md @@ -32,7 +32,7 @@ Micrograd demonstrated that automatic differentiation—the heart of modern ML **When to use micrograd**: Perfect 2-hour introduction before starting TinyTorch -**Connection to TinyTorch**: Module 05 (Autograd) teaches the same core concepts with systems engineering focus +**Connection to TinyTorch**: Module 06 (Autograd) teaches the same core concepts with systems engineering focus ### nanoGPT diff --git a/tinytorch/site/resources.md b/tinytorch/site/resources.md index 955d511cc8..5ba974a0b7 100644 --- a/tinytorch/site/resources.md +++ b/tinytorch/site/resources.md @@ -21,7 +21,7 @@ - The book explains *why* PyTorch, TensorFlow, and JAX make certain design decisions - Together, they provide both hands-on implementation and theoretical understanding -**When to use it**: Read in parallel with TinyTorch. When you implement Module 05 (Autograd), read the book's chapter on automatic differentiation to understand the systems engineering behind your code. +**When to use it**: Read in parallel with TinyTorch. When you implement Module 06 (Autograd), read the book's chapter on automatic differentiation to understand the systems engineering behind your code. ## Related Academic Courses diff --git a/tinytorch/site/tito/data.md b/tinytorch/site/tito/data.md index c2fd181583..fb79ec9e0a 100644 --- a/tinytorch/site/tito/data.md +++ b/tinytorch/site/tito/data.md @@ -395,7 +395,7 @@ Or restore from backup: 5. **Version control for code**: ```bash - git commit -m "Completed Module 05: Autograd" + git commit -m "Completed Module 05: DataLoader" ``` `.tito/` is gitignored - use git for code versions diff --git a/tinytorch/site/tito/modules.md b/tinytorch/site/tito/modules.md index 1c999b3813..a91524a204 100644 --- a/tinytorch/site/tito/modules.md +++ b/tinytorch/site/tito/modules.md @@ -198,7 +198,7 @@ tito module start 01 **Example**: ```bash -tito module start 05 # Start Module 05 (Autograd) +tito module start 05 # Start Module 05 (DataLoader) ``` Jupyter Lab opens with the generated notebook for Module 05 @@ -242,7 +242,7 @@ tito module complete 01 **Example**: ```bash -tito module complete 05 # Export Module 05 (Autograd) +tito module complete 05 # Export Module 05 (DataLoader) ``` **After exporting**: @@ -281,7 +281,7 @@ tito module status Module 02: Activations (completed 2025-11-16) Module 03: Layers (completed 2025-11-16) Module 04: Losses (not started) - Module 05: Autograd (not started) + Module 05: DataLoader (not started) Progress: 3/20 modules (15%) diff --git a/tinytorch/src/02_activations/ABOUT.md b/tinytorch/src/02_activations/ABOUT.md index b5029df3b0..8f284f47eb 100644 --- a/tinytorch/src/02_activations/ABOUT.md +++ b/tinytorch/src/02_activations/ABOUT.md @@ -119,12 +119,12 @@ probabilities = softmax(logits) # Converts to probability distribution (sums to To keep this module focused, you will **not** implement: -- Gradient computation (that's Module 05: Autograd - `backward()` methods are stubs for now) +- Gradient computation (that's Module 06: Autograd - `backward()` methods are stubs for now) - Learnable parameters (activations are fixed mathematical functions) - Advanced variants (LeakyReLU, ELU, Swish - PyTorch has dozens, you'll build the core five) - GPU acceleration (your NumPy implementation runs on CPU) -**You are building the nonlinear transformations.** Automatic differentiation comes in Module 05. +**You are building the nonlinear transformations.** Automatic differentiation comes in Module 06. ## API Reference @@ -144,7 +144,7 @@ class ActivationName: return self.forward(x) def backward(self, grad: Tensor) -> Tensor: - # Stub for Module 05 + # Stub for Module 06 pass ``` @@ -230,7 +230,7 @@ This simplicity is ReLU's greatest strength. The operation is a single compariso ReLU creates **sparsity**. When half your activations are exactly zero, computations become faster (multiplying by zero is free) and models generalize better (sparse representations are less prone to overfitting). In a 1000-neuron layer, ReLU typically activates 300-500 neurons, effectively creating a smaller, specialized network for each input. -The discontinuity at zero is both a feature and a bug. During training (Module 05), you'll discover that ReLU's gradient is exactly 1 for positive inputs and exactly 0 for negative inputs. This prevents the vanishing gradient problem that plagued sigmoid-based networks. But it creates a new problem: **dying ReLU**. If a neuron's weights shift such that it always receives negative inputs, it will output zero forever, and the zero gradient means it can never recover. +The discontinuity at zero is both a feature and a bug. During training (Module 08), you'll discover that ReLU's gradient is exactly 1 for positive inputs and exactly 0 for negative inputs. This prevents the vanishing gradient problem that plagued sigmoid-based networks. But it creates a new problem: **dying ReLU**. If a neuron's weights shift such that it always receives negative inputs, it will output zero forever, and the zero gradient means it can never recover. Despite this limitation, ReLU remains the default choice for hidden layers in CNNs and feedforward networks. Its speed and effectiveness at preventing vanishing gradients make it hard to beat. @@ -306,7 +306,7 @@ exp(x - max) / Σ exp(x - max) = exp(x) / Σ exp(x) Softmax amplifies differences. If the input is `[1, 2, 3]`, the output is approximately `[0.09, 0.24, 0.67]`. The largest input gets 67% of the probability mass, even though it's only 3× larger than the smallest input. This is because exponentials grow superlinearly. In classification, this is desirable: you want the network to be confident when it's confident. -But softmax's coupling is a gotcha. When you change one input, all outputs change because they're normalized by the same sum. This means the gradient involves a Jacobian matrix, not just element-wise derivatives. You'll see this complexity when you implement `backward()` in Module 05. +But softmax's coupling is a gotcha. When you change one input, all outputs change because they're normalized by the same sum. This means the gradient involves a Jacobian matrix, not just element-wise derivatives. You'll see this complexity when you implement `backward()` in Module 06. ### Choosing Activations @@ -357,7 +357,7 @@ Your TinyTorch activations and PyTorch's `torch.nn.functional` activations imple | **Backend** | NumPy (Python/C) | C++/CUDA kernels | | **Speed** | 1× (CPU baseline) | 10-100× faster (GPU) | | **Numerical Stability** | ✓ Max subtraction (Softmax), clipping (Sigmoid) | ✓ Same techniques | -| **Autograd** | Stubs (Module 05) | Full gradient computation | +| **Autograd** | Stubs (Module 06) | Full gradient computation | | **Variants** | 5 core activations | 30+ variants (LeakyReLU, PReLU, Mish, etc.) | ### Code Comparison @@ -440,7 +440,7 @@ A batch of 32 samples passes through a hidden layer with 4096 neurons and ReLU a 32 × 4096 × 4 bytes = **524,288 bytes ≈ 512 KB** -This is the activation memory for ONE layer. A 100-layer network needs 50 MB just to store activations for one forward pass. This is why activation memory dominates training memory usage (you'll see this in Module 05 when you cache activations for backpropagation). +This is the activation memory for ONE layer. A 100-layer network needs 50 MB just to store activations for one forward pass. This is why activation memory dominates training memory usage (you'll see this in Module 06 when you cache activations for backpropagation). ``` **Q2: Computational Cost** diff --git a/tinytorch/src/03_layers/03_layers.py b/tinytorch/src/03_layers/03_layers.py index 4e46ea11ba..be54e70814 100644 --- a/tinytorch/src/03_layers/03_layers.py +++ b/tinytorch/src/03_layers/03_layers.py @@ -167,7 +167,7 @@ Let's build our layer system step by step. We'll implement two essential layer t - All methods defined INSIDE classes (no monkey-patching) - Forward methods return new tensors, preserving immutability - parameters() method enables optimizer integration -- Gradient tracking will be added in Module 05 (Autograd) +- Gradient tracking will be added in Module 06 (Autograd) """ # %% [markdown] diff --git a/tinytorch/src/03_layers/ABOUT.md b/tinytorch/src/03_layers/ABOUT.md index 008390ec5a..3e14aa8748 100644 --- a/tinytorch/src/03_layers/ABOUT.md +++ b/tinytorch/src/03_layers/ABOUT.md @@ -113,8 +113,8 @@ output = layer2(x) To keep this module focused, you will **not** implement: -- Automatic gradient computation (that's Module 05: Autograd) -- Parameter optimization (that's Module 06: Optimizers) +- Automatic gradient computation (that's Module 06: Autograd) +- Parameter optimization (that's Module 07: Optimizers) - Hundreds of layer types (PyTorch has Conv2d, LSTM, Attention - you'll build Linear and Dropout) - Automatic training/eval mode switching (PyTorch's `model.train()` - you'll manually pass `training` flag) @@ -245,7 +245,7 @@ def __init__(self, in_features, out_features, bias=True): self.bias = None ``` -The `requires_grad=True` flag marks these tensors for gradient computation in Module 05. Even though you haven't built autograd yet, your layers are already prepared for it. Bias starts at zero because the weight initialization already handles the scale, and zero is a neutral starting point for per-class adjustments. +The `requires_grad=True` flag marks these tensors for gradient computation in Module 06. Even though you haven't built autograd yet, your layers are already prepared for it. Bias starts at zero because the weight initialization already handles the scale, and zero is a neutral starting point for per-class adjustments. For Linear(1000, 10), the scale is `sqrt(1/1000) ≈ 0.032`. For Linear(10, 1000), the scale is `sqrt(1/10) ≈ 0.316`. Layers with more inputs get smaller initial weights because each input contributes to the output, and you want their combined effect to remain stable. @@ -269,7 +269,7 @@ layer1 = Linear(784, 256) layer2 = Linear(256, 10) all_params = layer1.parameters() + layer2.parameters() -# In Module 06, you'll pass all_params to optimizer.step() +# In Module 07, you'll pass all_params to optimizer.step() ``` Each Linear layer independently manages its own parameters. The Sequential container extends this pattern by collecting parameters from all its contained layers, enabling hierarchical composition. diff --git a/tinytorch/src/04_losses/ABOUT.md b/tinytorch/src/04_losses/ABOUT.md index e6831bf115..2a2b72b221 100644 --- a/tinytorch/src/04_losses/ABOUT.md +++ b/tinytorch/src/04_losses/ABOUT.md @@ -47,7 +47,7 @@ Listen to an AI-generated overview. Loss functions are the mathematical conscience of machine learning. Every neural network needs to know when it's right and when it's wrong. Loss functions provide that feedback by measuring the distance between what your model predicts and what actually happened. Without loss functions, models have no way to improve - they're like athletes training without knowing their score. -In this module, you'll implement three essential loss functions: Mean Squared Error (MSE) for regression, Cross-Entropy for multi-class classification, and Binary Cross-Entropy for binary decisions. You'll also master the log-sum-exp trick, a crucial numerical stability technique that prevents computational overflow with large numbers. These implementations will serve as the foundation for Module 05: Autograd, where gradients flow backward from these loss values to update model parameters. +In this module, you'll implement three essential loss functions: Mean Squared Error (MSE) for regression, Cross-Entropy for multi-class classification, and Binary Cross-Entropy for binary decisions. You'll also master the log-sum-exp trick, a crucial numerical stability technique that prevents computational overflow with large numbers. These implementations will serve as the foundation for Module 06: Autograd, where gradients flow backward from these loss values to update model parameters. By the end, you'll understand not just how to compute loss, but why different problems require different loss functions, and how numerical stability shapes production ML systems. @@ -102,7 +102,7 @@ loss = criterion(predictions, targets) # Scalar feedback signal for learning To keep this module focused, you will **not** implement: -- Gradient computation (that's Module 05: Autograd) +- Gradient computation (that's Module 06: Autograd) - Advanced loss variants (Focal Loss, Label Smoothing, Huber Loss) - Hierarchical or sampled softmax for large vocabularies (PyTorch optimization) - Custom reduction strategies beyond mean @@ -172,7 +172,7 @@ Loss functions transform the abstract question "how good is my model?" into a co The key insight is that loss functions must be differentiable - you need to know not just the current error, but which direction to move parameters to reduce that error. This is why we use squared differences instead of absolute differences in MSE: the square function has a smooth derivative that points toward improvement. -Every training iteration follows the same pattern: forward pass produces predictions, loss function measures error, backward pass (Module 05) computes how to improve. The loss value itself becomes a single number summarizing model quality across an entire batch of examples. +Every training iteration follows the same pattern: forward pass produces predictions, loss function measures error, backward pass (Module 06) computes how to improve. The loss value itself becomes a single number summarizing model quality across an entire batch of examples. ### Mean Squared Error @@ -574,18 +574,18 @@ For students who want to understand the academic foundations and explore deeper: ## What's Next -```{seealso} Coming Up: Module 05 - Autograd +```{seealso} Coming Up: Module 05 - DataLoader -Implement automatic differentiation to compute gradients of your loss functions. You'll build the computational graph that tracks operations and use the chain rule to flow gradients backward through your network - the foundation of all deep learning optimization. +Build efficient data pipelines that handle batching, shuffling, and iteration over your datasets. DataLoader prepares your training data so that autograd and training loops can consume it efficiently. ``` **Preview - How Your Loss Functions Get Used in Future Modules:** | Module | What It Does | Your Loss In Action | |--------|--------------|---------------------| -| **05: Autograd** | Automatic differentiation | `loss.backward()` computes gradients | -| **06: Optimizers** | Parameter updates | `optimizer.step()` uses loss gradients to improve weights | -| **07: Training** | Complete training loop | `loss = criterion(outputs, targets)` measures progress | +| **06: Autograd** | Automatic differentiation | `loss.backward()` computes gradients | +| **07: Optimizers** | Parameter updates | `optimizer.step()` uses loss gradients to improve weights | +| **08: Training** | Complete training loop | `loss = criterion(outputs, targets)` measures progress | ## Get Started diff --git a/tinytorch/src/05_dataloader/05_dataloader.py b/tinytorch/src/05_dataloader/05_dataloader.py index 521e10fd3f..fa0c9138ee 100644 --- a/tinytorch/src/05_dataloader/05_dataloader.py +++ b/tinytorch/src/05_dataloader/05_dataloader.py @@ -17,19 +17,19 @@ # %% [markdown] """ -# Module 08: DataLoader - Efficient Data Pipeline for ML Training +# Module 05: DataLoader - Efficient Data Pipeline for ML Training -Welcome to Module 08! You're about to build the data loading infrastructure that transforms how ML models consume data during training. +Welcome to Module 05! You're about to build the data loading infrastructure that transforms how ML models consume data during training. ## 🔗 Prerequisites & Progress -**You've Built**: Tensor operations, activations, layers, losses, autograd, optimizers, and training loops +**You've Built**: Tensor operations, activations, layers, and losses **You'll Build**: Dataset abstraction, DataLoader with batching/shuffling, and real dataset support -**You'll Enable**: Efficient data pipelines that feed hungry neural networks with properly formatted batches +**You'll Enable**: Efficient data pipelines that will feed hungry neural networks with properly formatted batches **Connection Map**: ``` -Training Loop → DataLoader → Batched Data → Model -(Module 07) (Module 08) (optimized) (ready to learn) +Losses → DataLoader → Autograd → Optimizers → Training +(Module 04) (Module 05) (Module 06) (Module 07) (Module 08) ``` ## 🎯 Learning Objectives @@ -44,7 +44,7 @@ Let's transform scattered data into organized learning batches! ## 📦 Where This Code Lives in the Final Package -**Learning Side:** You work in `modules/08_dataloader/dataloader_dev.py` +**Learning Side:** You work in `modules/05_dataloader/dataloader_dev.py` **Building Side:** Code exports to `tinytorch.data.loader` ```python @@ -1147,7 +1147,7 @@ Now that you've built the DataLoader abstraction, you're ready to use it with re TinyTorch separates **mechanics** (this module) from **application** (examples/milestones): ``` -Module 08 (DataLoader) Examples & Milestones +Module 05 (DataLoader) Examples & Milestones ┌──────────────────────┐ ┌────────────────────────┐ │ Dataset abstraction │ │ Real MNIST digits │ │ TensorDataset impl │ ───> │ CIFAR-10 images │ @@ -1971,7 +1971,7 @@ Congratulations! You've built a complete data loading pipeline for ML training! ### Ready for Next Steps Your DataLoader implementation enables efficient training of CNNs and larger models with proper data pipeline management. -Export with: `tito export 08_dataloader` +Export with: `tito export 05_dataloader` **Apply your knowledge:** - Milestone 03: Train MLP on real MNIST digits diff --git a/tinytorch/src/06_autograd/06_autograd.py b/tinytorch/src/06_autograd/06_autograd.py index cf6ead6b96..0784b0233f 100644 --- a/tinytorch/src/06_autograd/06_autograd.py +++ b/tinytorch/src/06_autograd/06_autograd.py @@ -14,19 +14,19 @@ # %% [markdown] """ -# Module 05: Autograd ⚡ - The Gradient Engine +# Module 06: Autograd ⚡ - The Gradient Engine -Welcome to Module 05! Today you'll awaken the gradient engine and unlock automatic differentiation. +Welcome to Module 06! Today you'll awaken the gradient engine and unlock automatic differentiation. ## 🔗 Prerequisites & Progress -**You've Built**: Tensor operations, activations, layers, and loss functions +**You've Built**: Tensor operations, activations, layers, losses, and DataLoader **You'll Build**: The autograd system that computes gradients automatically **You'll Enable**: Learning! Training! The ability to optimize neural networks! **Connection Map**: ``` -Modules 01-04 → Autograd → Training (Module 06-07) -(forward pass) (backward pass) (learning loops) +Modules 01-05 → Autograd → Optimizers → Training +(forward pass) (Module 06) (Module 07) (Module 08) ``` ## 🎯 Learning Objectives @@ -41,7 +41,7 @@ By the end of this module, you will: ## 📦 Where This Code Lives in the Final Package -**Learning Side:** You work in `modules/05_autograd/autograd_dev.py` +**Learning Side:** You work in `modules/06_autograd/autograd_dev.py` **Building Side:** Code exports to `tinytorch.core.autograd` ```python @@ -2385,7 +2385,7 @@ If your gradients look wrong or you get mysterious errors: PyTorch explicitly disables gradient tracking during parameter updates to allow safe in-place operations: ```python -# PyTorch pattern (we'll implement this in Module 06: Optimizers) +# PyTorch pattern (we'll implement this in Module 07: Optimizers) with torch.no_grad(): W -= 0.01 * W.grad # Safe inside no_grad context ``` @@ -2549,7 +2549,7 @@ def test_module(): print("\n" + "=" * 50) print("🎉 ALL TESTS PASSED! Module ready for export.") - print("Run: tito module complete 05_autograd") + print("Run: tito module complete 06_autograd") # Test function defined above, will be called in main block @@ -2659,7 +2659,7 @@ After answering these questions, consider: 3. **How does this connect to Module 01?** Why did we include requires_grad, grad, and backward() from the start? 4. **What production patterns emerged?** What choices would you make differently for a research prototype vs. production system? -These questions prepare you for Module 06 (Optimizers), where you'll use these gradients to actually update parameters and train models! +These questions prepare you for Module 07 (Optimizers), where you'll use these gradients to actually update parameters and train models! """ # %% [markdown] @@ -2738,9 +2738,9 @@ print(f"x.grad: {x.grad}") # Gradient w.r.t. x print(f"W.grad: {W.grad}") # Gradient w.r.t. W ``` -Export with: `tito module complete 05_autograd` +Export with: `tito module complete 06_autograd` -**Next**: Module 06 will add optimizers (SGD, Adam) that use these gradients to actually train neural networks! 🎯 +**Next**: Module 07 will add optimizers (SGD, Adam) that use these gradients to actually train neural networks! 🎯 ### 📈 Progress: Autograd ✓ ``` @@ -2748,8 +2748,8 @@ Export with: `tito module complete 05_autograd` ✅ Module 02: Activations (Non-linearities) ✅ Module 03: Layers (Building blocks) ✅ Module 04: Losses (Training objectives) -✅ Module 05: Autograd (Gradient engine) ← YOU ARE HERE -🔄 Module 06: Optimizers (Learning algorithms) -🔄 Module 07: Training (Complete training loops) +✅ Module 06: Autograd (Gradient engine) ← YOU ARE HERE +🔄 Module 07: Optimizers (Learning algorithms) +🔄 Module 08: Training (Complete training loops) ``` """ diff --git a/tinytorch/src/07_optimizers/07_optimizers.py b/tinytorch/src/07_optimizers/07_optimizers.py index 4638467a60..796d3710fe 100644 --- a/tinytorch/src/07_optimizers/07_optimizers.py +++ b/tinytorch/src/07_optimizers/07_optimizers.py @@ -14,19 +14,19 @@ # %% [markdown] """ -# Module 06: Optimizers - Sophisticated Learning Algorithms +# Module 07: Optimizers - Sophisticated Learning Algorithms -Welcome to Module 06! You'll build optimizers that enable neural networks to learn from gradients using sophisticated algorithms. +Welcome to Module 07! You'll build optimizers that enable neural networks to learn from gradients using sophisticated algorithms. ## 🔗 Prerequisites & Progress -**You've Built**: Tensor with gradients (Modules 01-05) +**You've Built**: Tensor with gradients (Modules 01-06) **You'll Build**: SGD, Adam, and AdamW optimizers with sophisticated momentum and adaptive learning **You'll Enable**: Modern optimization algorithms that power state-of-the-art neural networks **Connection Map**: ``` Gradients → Optimizers → Training -(Module 05) (Module 06) (Module 07) +(Module 06) (Module 07) (Module 08) ``` ## 🎯 Learning Objectives @@ -40,7 +40,7 @@ Let's get started! ## 📦 Where This Code Lives in the Final Package -**Learning Side:** You work in `modules/06_optimizers/optimizers_dev.py` +**Learning Side:** You work in `modules/07_optimizers/optimizers_dev.py` **Building Side:** Code exports to `tinytorch.core.optimizers` ```python @@ -52,7 +52,7 @@ from tinytorch.core.optimizers import SGD, Adam, AdamW - **Learning:** Complete optimization system for modern neural network training - **Production:** Proper organization like PyTorch's torch.optim with all optimization algorithms together - **Consistency:** All optimization logic and parameter updating in core.optimizers -- **Integration:** Works seamlessly with gradients from Module 05 for complete training capability +- **Integration:** Works seamlessly with gradients from Module 06 for complete training capability """ # %% nbgrader={"grade": false, "grade_id": "imports", "solution": true} @@ -62,11 +62,11 @@ from tinytorch.core.optimizers import SGD, Adam, AdamW import numpy as np from typing import List, Union, Optional, Dict, Any -# Import Tensor from Module 01 (now with gradient support from Module 05) +# Import Tensor from Module 01 (now with gradient support from Module 06) from tinytorch.core.tensor import Tensor # Enable autograd to add gradient tracking to Tensor -# This module depends on Module 05 (Autograd) being available +# This module depends on Module 06 (Autograd) being available from tinytorch.core.autograd import enable_autograd enable_autograd() @@ -457,7 +457,7 @@ class SGD(Optimizer): Check if this optimizer uses momentum. This explicit API method replaces the need for hasattr() checks - in checkpointing code (Module 07). + in checkpointing code (Module 08). Returns: bool: True if momentum is enabled (momentum > 0), False otherwise @@ -1452,7 +1452,7 @@ def test_module(): print("\n" + "=" * 50) print("🎉 ALL TESTS PASSED! Module ready for export.") - print("Run: tito module complete 06_optimizers") + print("Run: tito module complete 07_optimizers") # %% [markdown] """ @@ -1580,9 +1580,9 @@ Congratulations! You've built sophisticated optimization algorithms that power m - All tests pass ✅ (validated by `test_module()`) ### Ready for Next Steps -Your optimizer implementations enable sophisticated neural network training! With gradients from Module 05 and optimizers from Module 06, you're ready to build complete training loops. +Your optimizer implementations enable sophisticated neural network training! With gradients from Module 06 and optimizers from Module 07, you're ready to build complete training loops. -Export with: `tito module complete 06_optimizers` +Export with: `tito module complete 07_optimizers` -**Next**: Module 07 will add training loops, learning rate scheduling, and checkpointing for complete end-to-end neural network training! +**Next**: Module 08 will add training loops, learning rate scheduling, and checkpointing for complete end-to-end neural network training! """ diff --git a/tinytorch/src/07_optimizers/ABOUT.md b/tinytorch/src/07_optimizers/ABOUT.md index efdb45b3be..ca71188d07 100644 --- a/tinytorch/src/07_optimizers/ABOUT.md +++ b/tinytorch/src/07_optimizers/ABOUT.md @@ -399,7 +399,7 @@ Let's walk through each line to understand the comparison: - **Line 1 (Import)**: TinyTorch exposes optimizers from `tinytorch.core.optimizers`; PyTorch uses `torch.optim`. The namespace structure mirrors production frameworks. - **Line 4 (Creation)**: Both use identical syntax: `Adam(model.parameters(), lr=0.001)`. The `model.parameters()` method returns an iterable of tensors with `requires_grad=True`. -- **Line 7-8 (Training)**: The loss computation and backward pass are identical. Your autograd system from Module 05 computes gradients just like PyTorch. +- **Line 7-8 (Training)**: The loss computation and backward pass are identical. Your autograd system from Module 06 computes gradients just like PyTorch. - **Line 9 (Update)**: Both call `optimizer.step()` to update parameters using computed gradients. The update rules are mathematically identical. - **Line 10 (Clear)**: Both call `optimizer.zero_grad()` to clear gradients before the next iteration. Without this, gradients would accumulate across batches. diff --git a/tinytorch/src/08_training/08_training.py b/tinytorch/src/08_training/08_training.py index 9f4038cdaa..273afc3778 100644 --- a/tinytorch/src/08_training/08_training.py +++ b/tinytorch/src/08_training/08_training.py @@ -14,19 +14,19 @@ # %% [markdown] """ -# Module 07: Training - Complete Learning Loops +# Module 08: Training - Complete Learning Loops -Welcome to Module 07! You're about to build the complete training infrastructure that brings neural networks to life through end-to-end learning. +Welcome to Module 08! You're about to build the complete training infrastructure that brings neural networks to life through end-to-end learning. ## 🔗 Prerequisites & Progress -**You've Built**: Tensors, activations, layers, losses, gradients, and optimizers +**You've Built**: Tensors, activations, layers, losses, DataLoader, gradients, and optimizers **You'll Build**: Complete training loops with checkpointing, scheduling, and gradient management **You'll Enable**: Full model training pipeline for the MLP milestone **Connection Map**: ``` -Optimizers (Module 06) → Training (Module 07) → DataLoader (Module 08) -(parameter updates) (complete loops) (efficient batching) +DataLoader → Autograd → Optimizers → Training → Convolutions +(Module 05) (Module 06) (Module 07) (Module 08) (Module 09) ``` ## 🎯 Learning Objectives @@ -41,7 +41,7 @@ Let's get started! ## 📦 Where This Code Lives in the Final Package -**Learning Side:** You work in `modules/07_training/training_dev.py` +**Learning Side:** You work in `modules/08_training/training_dev.py` **Building Side:** Code exports to `tinytorch.core.training` ```python @@ -722,7 +722,7 @@ class Trainer: state = {} # Trust optimizer has lr attribute (from Modules 06) state['lr'] = self.optimizer.lr - # Use explicit API for momentum state (Module 06) + # Use explicit API for momentum state (Module 07) # All optimizers with momentum support have get_momentum_state() method if hasattr(self.optimizer, 'has_momentum') and self.optimizer.has_momentum(): momentum_state = self.optimizer.get_momentum_state() @@ -735,7 +735,7 @@ class Trainer: if 'lr' in state: # Trust optimizer has lr attribute (from Modules 06) self.optimizer.lr = state['lr'] - # Use explicit API for momentum state (Module 06) + # Use explicit API for momentum state (Module 07) # All optimizers with momentum support have set_momentum_state() method if 'momentum_buffers' in state: if hasattr(self.optimizer, 'has_momentum') and self.optimizer.has_momentum(): @@ -795,7 +795,7 @@ def test_unit_trainer(): # Create trainer with REAL components model = SimpleModel() - optimizer = SGD(model.parameters(), lr=0.01) # Real SGD from Module 06 + optimizer = SGD(model.parameters(), lr=0.01) # Real SGD from Module 07 loss_fn = MSELoss() # Real MSELoss from Module 04 scheduler = CosineSchedule(max_lr=0.1, min_lr=0.01, total_epochs=10) @@ -1179,7 +1179,7 @@ def test_module(): # Create integrated system with REAL components model = SimpleModel() - optimizer = SGD(model.parameters(), lr=0.01) # Real SGD from Module 06 + optimizer = SGD(model.parameters(), lr=0.01) # Real SGD from Module 07 loss_fn = MSELoss() # Real MSELoss from Module 04 scheduler = CosineSchedule(max_lr=0.1, min_lr=0.001, total_epochs=3) @@ -1371,7 +1371,7 @@ Your training implementation enables sophisticated model training with proper sc **Export with:** `tito module complete 07` -**Next**: Module 08 will add DataLoader for efficient data pipeline management, completing the full training infrastructure needed for the MLP milestone! +**Next**: Module 09 (Convolutions) will add spatial neural network operations, enabling CNN architectures for computer vision! **🎓 You now understand the complete training infrastructure that powers modern ML systems!** """ diff --git a/tinytorch/src/09_convolutions/ABOUT.md b/tinytorch/src/09_convolutions/ABOUT.md index f02ebd7c1d..477bf00325 100644 --- a/tinytorch/src/09_convolutions/ABOUT.md +++ b/tinytorch/src/09_convolutions/ABOUT.md @@ -7,7 +7,7 @@ **Prerequisites: Modules 01-08** assumes you have: - Built the complete training pipeline (Modules 01-07) -- Implemented DataLoader for batch processing (Module 08) +- Implemented DataLoader for batch processing (Module 05) - Understanding of parameter initialization, forward/backward passes, and optimization If you can train an MLP on MNIST using your training loop and DataLoader, you're ready. diff --git a/tinytorch/src/12_attention/ABOUT.md b/tinytorch/src/12_attention/ABOUT.md index 4d1a20bd1e..1884940c21 100644 --- a/tinytorch/src/12_attention/ABOUT.md +++ b/tinytorch/src/12_attention/ABOUT.md @@ -9,7 +9,7 @@ - Tensor operations and shape manipulation (Module 01) - Activations, particularly softmax (Module 02) - Linear layers and weight projections (Module 03) -- Autograd for gradient computation (Module 05) +- Autograd for gradient computation (Module 06) - Tokenization and embeddings (Modules 10-11) If you can explain why `softmax(x).sum(axis=-1)` equals 1.0 and how embeddings convert token IDs to dense vectors, you're ready. diff --git a/tinytorch/src/14_profiling/14_profiling.py b/tinytorch/src/14_profiling/14_profiling.py index fe9e9c678f..c654acf919 100644 --- a/tinytorch/src/14_profiling/14_profiling.py +++ b/tinytorch/src/14_profiling/14_profiling.py @@ -32,7 +32,7 @@ All Modules (01-13) → Profiling (14) → Optimization Techniques (15-18) **Before starting this module, verify:** - [ ] Module 01 (Tensor): Core tensor operations - [ ] Module 03 (Layers): Linear layer implementation -- [ ] Module 08 (Spatial): Convolutional operations +- [ ] Module 09 (Spatial): Convolutional operations This module can work standalone with minimal Tensor implementation, but full functionality requires previous modules for realistic profiling scenarios diff --git a/tinytorch/src/18_acceleration/ABOUT.md b/tinytorch/src/18_acceleration/ABOUT.md index ba37055a34..498910ca2d 100644 --- a/tinytorch/src/18_acceleration/ABOUT.md +++ b/tinytorch/src/18_acceleration/ABOUT.md @@ -8,7 +8,7 @@ **Prerequisites: Modules 01-14** means you need: - Tensor operations (Module 01) for understanding data structures - Neural network layers (Module 03) for knowing what to accelerate -- Training loops (Module 07) for understanding the performance context +- Training loops (Module 08) for understanding the performance context - Profiling tools (Module 14) for measuring acceleration gains If you can multiply matrices and understand why matrix multiplication is expensive, you're ready. diff --git a/tinytorch/src/20_capstone/20_capstone.py b/tinytorch/src/20_capstone/20_capstone.py index 445bca06d2..8209efe706 100644 --- a/tinytorch/src/20_capstone/20_capstone.py +++ b/tinytorch/src/20_capstone/20_capstone.py @@ -1564,9 +1564,9 @@ Data → Tensor (M01) → Layers (M03) → Model → Training (M07) 1. **Dependency Management** - Each module imports from previous modules, creating a proper dependency graph 2. **API Consistency** - Tensor operations work the same whether in Module 01 or Module 18 3. **Composability** - Complex systems (transformers) built from simple primitives (linear layers) -4. **Progressive Enhancement** - Module 05 activated gradients dormant since Module 01 +4. **Progressive Enhancement** - Module 06 activated gradients dormant since Module 01 -**Reflection Question:** When you imported `from tinytorch.core.tensor import Tensor` in Module 15 (Quantization), the Tensor already had gradient tracking from Module 05. How does this "single source of truth" design simplify system integration compared to having separate BasicTensor and GradTensor classes? +**Reflection Question:** When you imported `from tinytorch.core.tensor import Tensor` in Module 15 (Quantization), the Tensor already had gradient tracking from Module 06. How does this "single source of truth" design simplify system integration compared to having separate BasicTensor and GradTensor classes? ### Benchmarking Methodology: Science Meets Engineering diff --git a/tinytorch/tests/03_layers/test_03_progressive_integration.py b/tinytorch/tests/03_layers/test_03_progressive_integration.py index b8509a5cf5..3ea045be4b 100644 --- a/tinytorch/tests/03_layers/test_03_progressive_integration.py +++ b/tinytorch/tests/03_layers/test_03_progressive_integration.py @@ -9,7 +9,7 @@ This is where we create reusable building blocks for neural networks. - Module 03: Layer base class and interface design - Integration: Layers work with tensors and activations from previous modules - Regression: Previous modules (01→03) still work correctly -- Preparation: Foundation ready for Module 05 (Dense networks) +- Preparation: Foundation ready for Module 05 (DataLoader) 💡 FOR STUDENTS: If tests fail, check: 1. Does your Layer base class exist in tinytorch.core.layers? @@ -417,7 +417,7 @@ class TestProgressiveStackIntegration: - Layer returns Tensor objects - Tensor operations work within layers - 💡 This is crucial for Module 05 (Dense layers) + 💡 This is crucial for Module 05 (DataLoader layers) """ try: from tinytorch.core.layers import Layer @@ -554,7 +554,7 @@ class TestProgressiveStackIntegration: - Layers (04) working - All components work together - 🎯 MILESTONE: Ready for Module 05 (Dense Networks)! + 🎯 MILESTONE: Ready for Module 05 (DataLoader Networks)! """ try: # Import all components @@ -625,7 +625,7 @@ class TestNeuralNetworkReadiness: """ 🧠 NEURAL NETWORK READINESS: Test foundation is ready for real neural networks. - 💡 These tests verify you have everything needed for Module 05 (Dense Networks). + 💡 These tests verify you have everything needed for Module 05 (DataLoader Networks). 🎯 Goal: Confirm the foundation supports building actual neural networks. """ @@ -775,7 +775,7 @@ class TestNeuralNetworkReadiness: - Activation functions - Output generation - 🎯 MILESTONE: Ready for Module 05 Dense Networks! + 🎯 MILESTONE: Ready for Module 05 DataLoader Networks! """ try: from tinytorch.core.tensor import Tensor @@ -873,7 +873,7 @@ class TestModuleCompletionReadiness: □ Integration with previous modules □ Ready for inheritance (Dense, Conv2D, etc.) - 🎯 SUCCESS = Ready for Module 05: Dense Networks! + 🎯 SUCCESS = Ready for Module 05: DataLoader! """ completion_checklist = { "Layer base class exists": False, @@ -979,7 +979,7 @@ class TestModuleCompletionReadiness: ✅ Integration with foundation ✅ Ready for neural networks - 🚀 READY FOR MODULE 05: DENSE NETWORKS! + 🚀 READY FOR MODULE 05: DATALOADER! 💡 What you can now do: - Inherit from Layer to create Dense layers diff --git a/tinytorch/tests/03_layers/test_progressive_integration.py b/tinytorch/tests/03_layers/test_progressive_integration.py index b8509a5cf5..e676d1f2b5 100644 --- a/tinytorch/tests/03_layers/test_progressive_integration.py +++ b/tinytorch/tests/03_layers/test_progressive_integration.py @@ -9,7 +9,7 @@ This is where we create reusable building blocks for neural networks. - Module 03: Layer base class and interface design - Integration: Layers work with tensors and activations from previous modules - Regression: Previous modules (01→03) still work correctly -- Preparation: Foundation ready for Module 05 (Dense networks) +- Preparation: Foundation ready for Module 05 (DataLoader) 💡 FOR STUDENTS: If tests fail, check: 1. Does your Layer base class exist in tinytorch.core.layers? @@ -417,7 +417,7 @@ class TestProgressiveStackIntegration: - Layer returns Tensor objects - Tensor operations work within layers - 💡 This is crucial for Module 05 (Dense layers) + 💡 This is crucial for Module 05 (DataLoader) """ try: from tinytorch.core.layers import Layer @@ -554,7 +554,7 @@ class TestProgressiveStackIntegration: - Layers (04) working - All components work together - 🎯 MILESTONE: Ready for Module 05 (Dense Networks)! + 🎯 MILESTONE: Ready for Module 05 (DataLoader)! """ try: # Import all components @@ -625,7 +625,7 @@ class TestNeuralNetworkReadiness: """ 🧠 NEURAL NETWORK READINESS: Test foundation is ready for real neural networks. - 💡 These tests verify you have everything needed for Module 05 (Dense Networks). + 💡 These tests verify you have everything needed for Module 05 (DataLoader). 🎯 Goal: Confirm the foundation supports building actual neural networks. """ @@ -775,7 +775,7 @@ class TestNeuralNetworkReadiness: - Activation functions - Output generation - 🎯 MILESTONE: Ready for Module 05 Dense Networks! + 🎯 MILESTONE: Ready for Module 05 DataLoader! """ try: from tinytorch.core.tensor import Tensor @@ -873,7 +873,7 @@ class TestModuleCompletionReadiness: □ Integration with previous modules □ Ready for inheritance (Dense, Conv2D, etc.) - 🎯 SUCCESS = Ready for Module 05: Dense Networks! + 🎯 SUCCESS = Ready for Module 05: DataLoader! """ completion_checklist = { "Layer base class exists": False, @@ -979,7 +979,7 @@ class TestModuleCompletionReadiness: ✅ Integration with foundation ✅ Ready for neural networks - 🚀 READY FOR MODULE 05: DENSE NETWORKS! + 🚀 READY FOR MODULE 05: DATALOADER! 💡 What you can now do: - Inherit from Layer to create Dense layers diff --git a/tinytorch/tests/04_losses/test_04_progressive_integration.py b/tinytorch/tests/04_losses/test_04_progressive_integration.py index b69abc1b6b..fb8292feb9 100644 --- a/tinytorch/tests/04_losses/test_04_progressive_integration.py +++ b/tinytorch/tests/04_losses/test_04_progressive_integration.py @@ -1,8 +1,8 @@ """ Module 04: Progressive Integration Tests -Tests that Module 05 (Linear/Networks) works correctly AND that the entire foundation stack works. +Tests that Module 04 (Losses) works correctly AND that the entire foundation stack works. -DEPENDENCY CHAIN: 01_setup → 02_tensor → 03_activations → 04_layers → 05_dense +DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_dataloader This is the FOUNDATION MILESTONE - everything should work together for neural networks! """ diff --git a/tinytorch/tests/04_losses/test_dense_integration.py b/tinytorch/tests/04_losses/test_dense_integration.py index 9fcc9a4b60..fc1e49cfe3 100644 --- a/tinytorch/tests/04_losses/test_dense_integration.py +++ b/tinytorch/tests/04_losses/test_dense_integration.py @@ -18,7 +18,7 @@ def test_dense_module_integration(): warnings.filterwarnings("ignore") results = { - "module_name": "05_dense", + "module_name": "05_dataloader", "integration_type": "dense_validation", "tests": [], "success": True, diff --git a/tinytorch/tests/04_losses/test_loss_progressive_integration.py b/tinytorch/tests/04_losses/test_loss_progressive_integration.py index 946cc6fbb9..4c04ccfa89 100644 --- a/tinytorch/tests/04_losses/test_loss_progressive_integration.py +++ b/tinytorch/tests/04_losses/test_loss_progressive_integration.py @@ -58,7 +58,7 @@ class TestLossGradientFlow: assert not np.isnan(loss.data), "Loss should not be NaN" assert not np.isinf(loss.data), "Loss should not be Inf" - # Verify network parameters exist (ready for gradient flow in Module 05) + # Verify network parameters exist (ready for gradient flow in Module 06) assert hasattr(layer1, 'weight'), "Layer1 should have weight for gradients" assert hasattr(layer1, 'bias'), "Layer1 should have bias for gradients" assert hasattr(layer2, 'weight'), "Layer2 should have weight for gradients" @@ -374,7 +374,7 @@ class TestLossLayerIntegration: print(" - Module 01 (Tensor)") print(" - Module 02 (Activations)") print(" - Module 03 (Layers)") - print(" Ready for Module 05 (Autograd)!") + print(" Ready for Module 05 (DataLoader) and Module 06 (Autograd)!") except ImportError as e: print(f"⚠️ Loss-layer integration test skipped: {e}") @@ -510,7 +510,7 @@ def test_module_04_losses_integration(): print(" ✅ Complete pipeline integration") print(" ✅ Edge cases (zeros, perfect predictions)") print("\n🚀 Module 04 is ready for production use!") - print(" Next: Module 05 will add autograd for automatic differentiation\n") + print(" Next: Module 05 will add DataLoader for data pipelines, then Module 06 adds autograd\n") if __name__ == "__main__": diff --git a/tinytorch/tests/04_losses/test_progressive_integration.py b/tinytorch/tests/04_losses/test_progressive_integration.py index eb61b3217c..19c661f868 100644 --- a/tinytorch/tests/04_losses/test_progressive_integration.py +++ b/tinytorch/tests/04_losses/test_progressive_integration.py @@ -1,8 +1,8 @@ """ Module 04: Progressive Integration Tests -Tests that Module 05 (Dense/Networks) works correctly AND that the entire foundation stack works. +Tests that Module 04 (Losses) works correctly AND that the entire foundation stack works. -DEPENDENCY CHAIN: 01_setup → 02_tensor → 03_activations → 04_layers → 05_dense +DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_dataloader This is the FOUNDATION MILESTONE - everything should work together for neural networks! """ diff --git a/tinytorch/tests/05_dataloader/test_autograd_core.py b/tinytorch/tests/05_dataloader/test_autograd_core.py index 3dae43c4ed..bad7c6c28a 100644 --- a/tinytorch/tests/05_dataloader/test_autograd_core.py +++ b/tinytorch/tests/05_dataloader/test_autograd_core.py @@ -1,5 +1,5 @@ """ -Module 08: Autograd - Core Functionality Tests +Module 06: Autograd - Core Functionality Tests Tests automatic differentiation and computational graphs """ diff --git a/tinytorch/tests/05_dataloader/test_dataloader_core.py b/tinytorch/tests/05_dataloader/test_dataloader_core.py index 0e331dd4e1..79a56a84e8 100644 --- a/tinytorch/tests/05_dataloader/test_dataloader_core.py +++ b/tinytorch/tests/05_dataloader/test_dataloader_core.py @@ -1,5 +1,5 @@ """ -Module 08: DataLoader - Core Functionality Tests +Module 05: DataLoader - Core Functionality Tests ================================================= WHY DATALOADER MATTERS: diff --git a/tinytorch/tests/05_dataloader/test_progressive_integration.py b/tinytorch/tests/05_dataloader/test_progressive_integration.py index 8fedcebf9a..3aa0a6ee4c 100644 --- a/tinytorch/tests/05_dataloader/test_progressive_integration.py +++ b/tinytorch/tests/05_dataloader/test_progressive_integration.py @@ -1,15 +1,15 @@ """ -Module 08: Progressive Integration Tests -Tests that Module 08 (DataLoader) works correctly AND that Foundation tier (01→07) still works. +Module 05: Progressive Integration Tests +Tests that Module 05 (DataLoader) works correctly AND that Foundation tier (01→04) still works. -DEPENDENCY CHAIN: 01_tensor → ... → 07_training → 08_dataloader +DEPENDENCY CHAIN: 01_tensor → ... → 04_losses → 05_dataloader This is where we enable efficient data loading for training. 🎯 WHAT THIS TESTS: -- Module 08: Dataset abstraction, batching, shuffling, data pipelines +- Module 05: Dataset abstraction, batching, shuffling, data pipelines - Integration: DataLoader works with Foundation tier modules -- Regression: Complete ML pipeline (01→08) still works correctly -- Preparation: Ready for CNNs (Module 09) and advanced architectures +- Regression: Complete ML pipeline still works correctly +- Preparation: Ready for Autograd (Module 06) and subsequent training infrastructure 💡 FOR STUDENTS: If tests fail, check: 1. Does your Variable class exist in tinytorch.core.autograd? @@ -34,7 +34,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) class TestCompleteMLPipelineStillWorks: """ - 🔄 REGRESSION CHECK: Verify complete ML pipeline (01→08) still works after autograd development. + 🔄 REGRESSION CHECK: Verify ML pipeline works with DataLoader. 💡 If these fail: You may have broken something in the ML pipeline while implementing autograd. 🔧 Fix: Check that autograd doesn't interfere with basic forward pass functionality. @@ -115,15 +115,12 @@ class TestCompleteMLPipelineStillWorks: 🔍 IMPORT ERROR: {str(e)} 🔧 PIPELINE REQUIREMENTS: - All previous modules (01→08) must be working: + Modules 01-05 must be working: 1. Tensor operations (Module 01) 2. Activation functions (Module 02) 3. Layer infrastructure (Module 03) 4. Losses (Module 04) - 5. Autograd (Module 05) - 6. Optimizers (Module 06) - 7. Training loops (Module 07) - 8. Data loading (Module 08) + 5. Data loading (Module 05) 💡 DEBUG STEPS: 1. Test each module individually @@ -241,7 +238,7 @@ class TestCompleteMLPipelineStillWorks: class TestModule09AutogradCore: """ - 🆕 NEW FUNCTIONALITY: Test Module 09 (Autograd) core implementation. + 🆕 NEW FUNCTIONALITY: Test Module 06 (Autograd) core implementation. 💡 What you're implementing: Automatic differentiation for gradient-based learning. 🎯 Goal: Enable gradient computation for neural network training. @@ -766,7 +763,7 @@ class TestModule09Completion: □ Optimization readiness □ Memory efficient implementation - 🎯 SUCCESS = Ready for Module 10: Optimizers! + 🎯 SUCCESS = Ready for Module 07: Optimizers! """ autograd_capabilities = { "Tensor gradient tracking": False, diff --git a/tinytorch/tests/06_autograd/test_autograd_core.py b/tinytorch/tests/06_autograd/test_autograd_core.py index 3edcc444c6..6fd9da0a4f 100644 --- a/tinytorch/tests/06_autograd/test_autograd_core.py +++ b/tinytorch/tests/06_autograd/test_autograd_core.py @@ -1,5 +1,5 @@ """ -Module 05: Autograd - Core Functionality Tests +Module 06: Autograd - Core Functionality Tests =============================================== These tests verify automatic differentiation works correctly. diff --git a/tinytorch/tests/06_autograd/test_progressive_integration.py b/tinytorch/tests/06_autograd/test_progressive_integration.py index 392da82571..3e58c911f8 100644 --- a/tinytorch/tests/06_autograd/test_progressive_integration.py +++ b/tinytorch/tests/06_autograd/test_progressive_integration.py @@ -1,8 +1,8 @@ """ -Module 05: Progressive Integration Tests -Tests that Module 05 (Autograd) works correctly AND that the entire prior stack works. +Module 06: Progressive Integration Tests +Tests that Module 06 (Autograd) works correctly AND that the entire prior stack works. -DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_autograd +DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_dataloader → 06_autograd This is where we enable automatic differentiation for gradient-based learning. """ @@ -15,10 +15,10 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) class TestPriorStackStillWorking: - """Quick regression checks that prior modules (01→07) still work.""" + """Quick regression checks that prior modules (01→05) still work.""" def test_foundation_stack_stable(self): - """Verify foundation stack (01→05) remains stable.""" + """Verify foundation stack (01→06) remains stable.""" # Environment (Module 01) assert sys.version_info >= (3, 8), "Foundation broken: Python version" @@ -37,7 +37,7 @@ class TestPriorStackStillWorking: assert True, "Foundation not implemented yet" def test_advanced_stack_stable(self): - """Verify advanced modules (06→07) still work.""" + """Verify advanced modules (07→08) still work.""" try: from tinytorch.core.spatial import Conv2d as Conv2D from tinytorch.core.attention import MultiHeadAttention @@ -53,8 +53,8 @@ class TestPriorStackStillWorking: assert True, "Advanced stack not implemented yet" -class TestModule08DataLoaderCore: - """Test Module 08 (DataLoader) core functionality.""" +class TestModule05DataLoaderCore: + """Test Module 05 (DataLoader) core functionality.""" def test_dataset_creation(self): """Test basic dataset creation works.""" @@ -331,7 +331,7 @@ class TestRealWorldDataCapability: class TestRegressionPrevention: - """Ensure previous modules still work after Module 08 development.""" + """Ensure previous modules still work after Module 06 (Autograd) development.""" def test_no_foundation_regression(self): """Verify foundation stack (01→05) unchanged.""" diff --git a/tinytorch/tests/07_optimizers/test_optimizer_core.py b/tinytorch/tests/07_optimizers/test_optimizer_core.py index aa849afe6a..1f36a43dee 100644 --- a/tinytorch/tests/07_optimizers/test_optimizer_core.py +++ b/tinytorch/tests/07_optimizers/test_optimizer_core.py @@ -1,5 +1,5 @@ """ -Module 06: Optimizer Core Tests +Module 07: Optimizer Core Tests ================================ These tests verify that optimizers correctly update model parameters. @@ -21,8 +21,8 @@ WHAT WE TEST: CONNECTION TO OTHER MODULES: ---------------------------- - Uses Tensor (Module 01) - optimizers update tensor.data -- Uses autograd (Module 05) - optimizers read tensor.grad -- Enables Training (Module 07) - optimizers make learning possible +- Uses autograd (Module 06) - optimizers read tensor.grad +- Enables Training (Module 08) - optimizers make learning possible """ import pytest diff --git a/tinytorch/tests/07_optimizers/test_progressive_integration.py b/tinytorch/tests/07_optimizers/test_progressive_integration.py index f0d7c51184..d2e27b7199 100644 --- a/tinytorch/tests/07_optimizers/test_progressive_integration.py +++ b/tinytorch/tests/07_optimizers/test_progressive_integration.py @@ -1,15 +1,15 @@ """ -Module 06: Progressive Integration Tests -Tests that Module 06 (Optimizers) works correctly AND that the foundation stack (01→05) still works. +Module 07: Progressive Integration Tests +Tests that Module 07 (Optimizers) works correctly AND that the foundation stack (01→06) still works. -DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_autograd → 06_optimizers +DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_dataloader → 06_autograd → 07_optimizers This is where we enable learning through sophisticated optimization algorithms. 🎯 WHAT THIS TESTS: -- Module 06: SGD, Adam, AdamW optimizers with momentum and adaptive learning -- Integration: Optimizers work with gradients from autograd (Module 05) -- Regression: Foundation stack (01→05) still works correctly -- Preparation: Ready for training loops (Module 07) +- Module 07: SGD, Adam, AdamW optimizers with momentum and adaptive learning +- Integration: Optimizers work with gradients from autograd (Module 06) +- Regression: Foundation stack (01→06) still works correctly +- Preparation: Ready for training loops (Module 08) 💡 FOR STUDENTS: If tests fail, check: 1. Does your SGD class exist in tinytorch.core.optimizers? @@ -72,7 +72,7 @@ class TestFoundationStackStillWorks: def test_gradient_computation_stable(self): """ - ✅ TEST: Gradient computation from Module 05 still works + ✅ TEST: Gradient computation from Module 06 still works """ try: from tinytorch.core.tensor import Tensor @@ -104,7 +104,7 @@ class TestFoundationStackStillWorks: class TestModule06OptimizersCore: """ - 🆕 NEW FUNCTIONALITY: Test Module 06 (Optimizers) core implementation. + 🆕 NEW FUNCTIONALITY: Test Module 07 (Optimizers) core implementation. 💡 What you're implementing: Optimization algorithms that update parameters using gradients. 🎯 Goal: Enable neural networks to learn from their mistakes. @@ -400,7 +400,7 @@ class TestRegressionPrevention: f"Autograd broken. Expected {expected_grad}, got {x.grad.data}" except Exception as e: - assert False, f"Module 05 (autograd) broken: {str(e)}" + assert False, f"Module 06 (autograd) broken: {str(e)}" def test_progressive_compatibility(self): """Test that all foundation modules work together.""" diff --git a/tinytorch/tests/08_training/test_progressive_integration.py b/tinytorch/tests/08_training/test_progressive_integration.py index 9f07563821..6a4ad096d8 100644 --- a/tinytorch/tests/08_training/test_progressive_integration.py +++ b/tinytorch/tests/08_training/test_progressive_integration.py @@ -1,8 +1,8 @@ """ -Module 07: Progressive Integration Tests -Tests that Module 07 (Training) works correctly AND that the entire prior stack works. +Module 08: Progressive Integration Tests +Tests that Module 08 (Training) works correctly AND that the entire prior stack works. -DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_autograd → 06_optimizers → 07_training +DEPENDENCY CHAIN: 01_tensor → 02_activations → 03_layers → 04_losses → 05_dataloader → 06_autograd → 07_optimizers → 08_training This is where we enable complete training loops for neural networks. """ @@ -38,7 +38,7 @@ class TestPriorStackStillWorking: assert True, "Foundation not implemented yet" def test_autograd_stable(self): - """Verify Module 09 (Autograd) still works.""" + """Verify Module 06 (Autograd) still works.""" try: from tinytorch.core.autograd import Variable, backward from tinytorch.core.tensor import Tensor @@ -56,8 +56,8 @@ class TestPriorStackStillWorking: assert True, "Autograd not implemented yet" -class TestModule10OptimizersCore: - """Test Module 10 (Optimizers) core functionality.""" +class TestModule07OptimizersCore: + """Test Module 07 (Optimizers) core functionality.""" def test_sgd_optimizer_creation(self): """Test SGD optimizer creation and basic functionality.""" diff --git a/tinytorch/tests/08_training/test_training_core.py b/tinytorch/tests/08_training/test_training_core.py index fb0b540062..9b90c1f28a 100644 --- a/tinytorch/tests/08_training/test_training_core.py +++ b/tinytorch/tests/08_training/test_training_core.py @@ -1,5 +1,5 @@ """ -Module 07: Training - Core Functionality Tests +Module 08: Training - Core Functionality Tests =============================================== WHY TRAINING MATTERS: diff --git a/tinytorch/tests/09_convolutions/test_progressive_integration.py b/tinytorch/tests/09_convolutions/test_progressive_integration.py index 2d11b27530..62f8c8b9db 100644 --- a/tinytorch/tests/09_convolutions/test_progressive_integration.py +++ b/tinytorch/tests/09_convolutions/test_progressive_integration.py @@ -1,8 +1,8 @@ """ Module 09: Progressive Integration Tests -Tests that Module 09 (Convolutions) works correctly AND that Foundation + DataLoader work. +Tests that Module 09 (Convolutions) works correctly AND that Foundation + Training work. -DEPENDENCY CHAIN: 01_tensor → ... → 07_training → 08_dataloader → 09_convolutions +DEPENDENCY CHAIN: 01_tensor → ... → 05_dataloader → 06_autograd → 07_optimizers → 08_training → 09_convolutions This is where CNNs enable computer vision through spatial feature extraction. """ @@ -31,7 +31,7 @@ class TestPriorStackStillWorking: assert True, "Tensor foundation not implemented yet" def test_spatial_operations_stable(self): - """Verify Module 06 (Spatial) operations still work.""" + """Verify Module 09 (Convolutions/Spatial) operations still work.""" try: from tinytorch.core.spatial import Conv2d, MaxPool2d @@ -46,8 +46,8 @@ class TestPriorStackStillWorking: assert True, "Spatial operations not implemented yet" -class TestModule07AttentionCore: - """Test Module 07 (Attention) core functionality.""" +class TestModule12AttentionCore: + """Test Module 12 (Attention) core functionality.""" def test_attention_mechanism_creation(self): """Test basic attention mechanism works.""" diff --git a/tinytorch/tests/10_tokenization/WRONG_VS_CORRECT.md b/tinytorch/tests/10_tokenization/WRONG_VS_CORRECT.md index 56447ffbc0..26ef9556f1 100644 --- a/tinytorch/tests/10_tokenization/WRONG_VS_CORRECT.md +++ b/tinytorch/tests/10_tokenization/WRONG_VS_CORRECT.md @@ -44,7 +44,7 @@ class TestAdvancedTrainingFeatures: # ← WRONG MODULE! Module 10: Progressive Integration Tests Tests that Module 10 (Tokenization) works correctly... # ← CORRECT! -DEPENDENCY CHAIN: 01_tensor → ... → 08_dataloader → 10_tokenization → 11_embeddings # ← CORRECT! +DEPENDENCY CHAIN: 01_tensor → ... → 05_dataloader → ... → 08_training → 09_convolutions → 10_tokenization # ← CORRECT! This is where we enable text processing for NLP tasks. # ← CORRECT! """ diff --git a/tinytorch/tests/10_tokenization/test_progressive_integration.py b/tinytorch/tests/10_tokenization/test_progressive_integration.py index 0e3e33eb11..1a26f9a232 100644 --- a/tinytorch/tests/10_tokenization/test_progressive_integration.py +++ b/tinytorch/tests/10_tokenization/test_progressive_integration.py @@ -2,7 +2,7 @@ Module 10: Progressive Integration Tests Tests that Module 10 (Tokenization) works correctly AND that Foundation + Architecture tier work. -DEPENDENCY CHAIN: 01_tensor → ... → 08_dataloader → 09_convolutions → 10_tokenization +DEPENDENCY CHAIN: 01_tensor → ... → 05_dataloader → ... → 08_training → 09_convolutions → 10_tokenization This is where text processing begins for NLP pipelines. """ @@ -42,7 +42,7 @@ class TestPriorStackStillWorking: assert True, "ML pipeline not implemented yet" def test_optimization_stable(self): - """Verify Module 10 (Optimizers) still works.""" + """Verify Module 07 (Optimizers) still works.""" try: from tinytorch.core.optimizers import SGD, Adam from tinytorch.core.layers import Linear @@ -59,8 +59,8 @@ class TestPriorStackStillWorking: assert True, "Optimizers not implemented yet" -class TestModule11TrainingCore: - """Test Module 11 (Training) core functionality.""" +class TestModule08TrainingCore: + """Test Module 08 (Training) core functionality.""" def test_training_loop_creation(self): """Test basic training loop functionality.""" diff --git a/tinytorch/tests/11_embeddings/README.md b/tinytorch/tests/11_embeddings/README.md index 90e9b11e4a..e304ffd295 100644 --- a/tinytorch/tests/11_embeddings/README.md +++ b/tinytorch/tests/11_embeddings/README.md @@ -25,7 +25,7 @@ The file `test_progressive_integration.py` tests **Module 12 (Compression)** ins └──────────────┘ │ ▼ ┌──────────────┐ ┌─────────────┐ -│ Module 05 │ Gradient tracking │ Module 11 │ +│ Module 06 │ Gradient tracking │ Module 11 │ │ Autograd │◄──────────────────│ Embeddings │ └──────────────┘ └─────────────┘ ▲ diff --git a/tinytorch/tests/11_embeddings/test_training_integration.py b/tinytorch/tests/11_embeddings/test_training_integration.py index f28d678415..917ea9d189 100644 --- a/tinytorch/tests/11_embeddings/test_training_integration.py +++ b/tinytorch/tests/11_embeddings/test_training_integration.py @@ -1,5 +1,5 @@ """ -Module 11: Training - Integration Tests +Module 08: Training - Integration Tests Tests that complete training loops work with all system components """ diff --git a/tinytorch/tests/13_transformers/test_progressive_integration.py b/tinytorch/tests/13_transformers/test_progressive_integration.py index 37f9e8abdb..7aedfba991 100644 --- a/tinytorch/tests/13_transformers/test_progressive_integration.py +++ b/tinytorch/tests/13_transformers/test_progressive_integration.py @@ -143,10 +143,10 @@ class TestCompleteMLSystemStillWorks: 2. Activation functions (Module 02) 3. Layer infrastructure (Module 03) 4. Losses (Module 04) - 5. Autograd (Module 05) - 6. Optimizers (Module 06) - 7. Training loops (Module 07) - 8. Data loading (Module 08) + 5. Data loading (Module 05) + 6. Autograd (Module 06) + 7. Optimizers (Module 07) + 8. Training loops (Module 08) 9. Convolutions (Module 09) 10. Tokenization (Module 10) 11. Embeddings (Module 11) diff --git a/tinytorch/tests/17_memoization/test_progressive_integration.py b/tinytorch/tests/17_memoization/test_progressive_integration.py index 0a8fb9de6b..8188bd5979 100644 --- a/tinytorch/tests/17_memoization/test_progressive_integration.py +++ b/tinytorch/tests/17_memoization/test_progressive_integration.py @@ -581,16 +581,16 @@ class TestTinyTorchGraduationReadiness: # Students should understand PyTorch concepts because they built them: pytorch_concepts_mastered = { - 'torch.Tensor': 'Built in Module 02', - 'torch.nn.Module': 'Built in Module 04', - 'torch.nn.functional': 'Built in Module 03', - 'torch.optim': 'Built in Module 10', - 'torch.utils.data': 'Built in Module 08', - 'torch.autograd': 'Built in Module 09', - 'torch.nn.attention': 'Built in Module 07', - 'torch.jit': 'Built in Module 13 (kernels)', - 'torch.quantization': 'Built in Module 12', - 'torch.distributed': 'Built in Module 15 (MLOps)', + 'torch.Tensor': 'Built in Module 01', + 'torch.nn.Module': 'Built in Module 03', + 'torch.nn.functional': 'Built in Module 02', + 'torch.optim': 'Built in Module 07', + 'torch.utils.data': 'Built in Module 05', + 'torch.autograd': 'Built in Module 06', + 'torch.nn.attention': 'Built in Module 12', + 'torch.jit': 'Built in Module 18 (acceleration)', + 'torch.quantization': 'Built in Module 15', + 'torch.distributed': 'Built in Module 19 (MLOps)', } # They understand the systems behind the abstractions diff --git a/tinytorch/tests/20_capstone/README.md b/tinytorch/tests/20_capstone/README.md index a568aa8043..37243caf84 100644 --- a/tinytorch/tests/20_capstone/README.md +++ b/tinytorch/tests/20_capstone/README.md @@ -36,7 +36,7 @@ The capstone tests verify that all 19 previous modules work together to build pr ### Priority 6: Gradient Flow - **test_deep_network_gradient_flow**: Gradients flow through all layer types - **test_gradient_accumulation_correctness**: Shared parameters accumulate gradients -- Validates: Module 05 autograd across all modules +- Validates: Module 06 autograd across all modules ### Priority 7: Memory & Performance - **test_memory_efficiency**: Memory usage is reasonable diff --git a/tinytorch/tests/README.md b/tinytorch/tests/README.md index 1b30421d0e..fd9380966b 100644 --- a/tinytorch/tests/README.md +++ b/tinytorch/tests/README.md @@ -34,7 +34,7 @@ These tests validate that each module works correctly in isolation. **Philosophy**: When these tests fail, the error message should teach the student what went wrong and how to fix it. -### ⚡ Autograd Edge Cases (`05_autograd/`) +### ⚡ Autograd Edge Cases (`06_autograd/`) **Purpose**: Stress-test autograd system **Scope**: Autograd internals and edge cases **Files**: @@ -101,7 +101,7 @@ When adding a test, ask: - **Is it testing one module?** → Put in `XX_modulename/` - **Is it testing modules working together?** → Put in `integration/` - **Is it teaching debugging?** → Put in `debugging/` -- **Is it an autograd edge case?** → Put in `05_autograd/` +- **Is it an autograd edge case?** → Put in `06_autograd/` ## Most Important Tests diff --git a/tinytorch/tests/checkpoints/checkpoint_09_optimization.py b/tinytorch/tests/checkpoints/checkpoint_09_optimization.py index c0dd065ec7..4ddc4c738a 100644 --- a/tinytorch/tests/checkpoints/checkpoint_09_optimization.py +++ b/tinytorch/tests/checkpoints/checkpoint_09_optimization.py @@ -1,5 +1,5 @@ """ -Checkpoint 9: Optimization (After Module 10 - Optimizers) +Checkpoint 9: Optimization (After Module 07 - Optimizers) Question: "Can I optimize neural networks with sophisticated algorithms?" """ diff --git a/tinytorch/tests/checkpoints/checkpoint_10_training.py b/tinytorch/tests/checkpoints/checkpoint_10_training.py index a90b6bb306..d870e8386a 100644 --- a/tinytorch/tests/checkpoints/checkpoint_10_training.py +++ b/tinytorch/tests/checkpoints/checkpoint_10_training.py @@ -1,5 +1,5 @@ """ -Checkpoint 10: Training (After Module 11 - Training) +Checkpoint 10: Training (After Module 08 - Training) Question: "Can I build complete training loops for end-to-end learning?" """ diff --git a/tinytorch/tests/integration/integration_cnn_test.py b/tinytorch/tests/integration/integration_cnn_test.py index 8ca60eae0b..e21655e1f7 100644 --- a/tinytorch/tests/integration/integration_cnn_test.py +++ b/tinytorch/tests/integration/integration_cnn_test.py @@ -9,7 +9,7 @@ Required modules: - Module 01-08: Core MLP functionality (from MNIST test) - Module 09: Convolutions (Conv2d, MaxPool2d) - Module 10: DataLoader for efficient batch processing -- Module 11: CNN training capabilities +- Module 08: Training capabilities for CNN This demonstrates the milestone: "Can train CNNs on CIFAR-10" """ diff --git a/tinytorch/tests/integration/module_complete_orchestrator.py b/tinytorch/tests/integration/module_complete_orchestrator.py index f65182062d..400012d8d7 100644 --- a/tinytorch/tests/integration/module_complete_orchestrator.py +++ b/tinytorch/tests/integration/module_complete_orchestrator.py @@ -178,7 +178,7 @@ def main(): import argparse parser = argparse.ArgumentParser(description="Complete module workflow") - parser.add_argument("module", help="Module name (e.g., 05_dense)") + parser.add_argument("module", help="Module name (e.g., 05_dataloader)") parser.add_argument("--skip-test", action="store_true", help="Skip integration tests") diff --git a/tinytorch/tests/integration/run_integration_tests.py b/tinytorch/tests/integration/run_integration_tests.py index 5f013bff8a..ecb335d928 100644 --- a/tinytorch/tests/integration/run_integration_tests.py +++ b/tinytorch/tests/integration/run_integration_tests.py @@ -213,7 +213,7 @@ def main(): import argparse parser = argparse.ArgumentParser(description="Run TinyTorch integration tests") - parser.add_argument("module", help="Module number (e.g., 05_dense)") + parser.add_argument("module", help="Module number (e.g., 05_dataloader)") parser.add_argument("--quiet", action="store_true", help="Suppress detailed output") args = parser.parse_args() @@ -233,5 +233,5 @@ if __name__ == "__main__": else: # Default: run module 05 tests as example runner = IntegrationTestRunner() - report = runner.run_module_tests("05_dense") + report = runner.run_module_tests("05_dataloader") runner.print_report(report) diff --git a/tinytorch/tests/integration/run_module_tests.py b/tinytorch/tests/integration/run_module_tests.py index 0b3c351476..c04e6de182 100644 --- a/tinytorch/tests/integration/run_module_tests.py +++ b/tinytorch/tests/integration/run_module_tests.py @@ -203,7 +203,7 @@ def main(): import argparse parser = argparse.ArgumentParser(description="Run module integration tests") - parser.add_argument("module", help="Module name (e.g., 05_dense)") + parser.add_argument("module", help="Module name (e.g., 05_dataloader)") parser.add_argument("--json", action="store_true", help="Output JSON report") args = parser.parse_args() diff --git a/tinytorch/tests/integration/test_dataloader_integration.py b/tinytorch/tests/integration/test_dataloader_integration.py index 4e27772332..b3d1cebb3e 100644 --- a/tinytorch/tests/integration/test_dataloader_integration.py +++ b/tinytorch/tests/integration/test_dataloader_integration.py @@ -18,7 +18,7 @@ try: from tinytorch.data.loader import Dataset, TensorDataset, DataLoader except (ImportError, ModuleNotFoundError): # Module not exported yet, use dev version - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'modules', 'source', '08_dataloader')) + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'modules', 'source', '05_dataloader')) from dataloader_dev import Dataset, TensorDataset, DataLoader sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'modules', 'source', '01_tensor')) from tensor_dev import Tensor diff --git a/tinytorch/tests/integration/test_integration_09_autograd.py b/tinytorch/tests/integration/test_integration_09_autograd.py index c4059ceeda..282256c58b 100644 --- a/tinytorch/tests/integration/test_integration_09_autograd.py +++ b/tinytorch/tests/integration/test_integration_09_autograd.py @@ -1,5 +1,5 @@ """ -Integration test for Module 09: Autograd +Integration test for Module 06: Autograd Validates that the autograd module integrates correctly with the TinyTorch package. This is a quick validation test, not a comprehensive capability test. diff --git a/tinytorch/tests/integration/test_module_05_dense.py b/tinytorch/tests/integration/test_module_05_dense.py index 7816a26942..a6314879d1 100644 --- a/tinytorch/tests/integration/test_module_05_dense.py +++ b/tinytorch/tests/integration/test_module_05_dense.py @@ -1,5 +1,5 @@ """ -Integration Tests for Module 05: Dense/Networks +Integration Tests for Module 05: DataLoader These tests verify that the module exports correctly and works as expected. Run with pytest for detailed reporting. """ diff --git a/tinytorch/tests/integration/test_module_dependencies.py b/tinytorch/tests/integration/test_module_dependencies.py index 773d27ebce..c4aee59225 100644 --- a/tinytorch/tests/integration/test_module_dependencies.py +++ b/tinytorch/tests/integration/test_module_dependencies.py @@ -51,20 +51,20 @@ def get_module_integration_tests(module_name: str): if "04_layers" in deps: tests.append(("test_layer_integration", test_layer_integration)) - if "05_dense" in deps: - tests.append(("test_dense_integration", test_dense_integration)) + if "05_dataloader" in deps: + tests.append(("test_dataloader_integration", test_dataloader_integration)) - if "10_autograd" in deps: + if "06_autograd" in deps: tests.append(("test_autograd_integration", test_autograd_integration)) - if "11_optimizers" in deps: + if "07_optimizers" in deps: tests.append(("test_optimizer_integration", test_optimizer_integration)) # Module-specific integration tests - if module_name == "05_dense": - tests.append(("test_dense_with_tensor", test_dense_with_tensor)) - tests.append(("test_dense_with_activations", test_dense_with_activations)) - tests.append(("test_multi_layer_network", test_multi_layer_network)) + if module_name == "05_dataloader": + tests.append(("test_dataloader_with_tensor", test_dataloader_with_tensor)) + tests.append(("test_dataloader_with_batching", test_dataloader_with_batching)) + tests.append(("test_dataloader_pipeline", test_dataloader_pipeline)) elif module_name == "09_convolutions": tests.append(("test_conv2d_with_tensor", test_conv2d_with_tensor)) diff --git a/tinytorch/tests/integration/test_optimizers_integration.py b/tinytorch/tests/integration/test_optimizers_integration.py index 7f0976a317..823ec09575 100644 --- a/tinytorch/tests/integration/test_optimizers_integration.py +++ b/tinytorch/tests/integration/test_optimizers_integration.py @@ -5,7 +5,7 @@ Tests that optimizers correctly integrate with: - Module 01: Tensor operations - Module 02: Activation functions - Module 03: Layers (Linear, Sequential) -- Module 05: Autograd (Tensor with gradients) +- Module 06: Autograd (Tensor with gradients) - Module 04: Losses (MSE, CrossEntropy) """ diff --git a/tinytorch/tests/integration/test_runner.py b/tinytorch/tests/integration/test_runner.py index 9ddebf41e3..d9f19480ca 100644 --- a/tinytorch/tests/integration/test_runner.py +++ b/tinytorch/tests/integration/test_runner.py @@ -203,8 +203,8 @@ def main(): import argparse parser = argparse.ArgumentParser(description="Run TinyTorch integration tests") - parser.add_argument("module", nargs='?', default="05_dense", - help="Module number (e.g., 05_dense)") + parser.add_argument("module", nargs='?', default="05_dataloader", + help="Module number (e.g., 05_dataloader)") parser.add_argument("--save", action="store_true", help="Save report to JSON file") parser.add_argument("--quiet", action="store_true", diff --git a/tinytorch/tests/integration_cnn_test.py b/tinytorch/tests/integration_cnn_test.py index e27692aa2a..d2f17c69b1 100644 --- a/tinytorch/tests/integration_cnn_test.py +++ b/tinytorch/tests/integration_cnn_test.py @@ -9,7 +9,7 @@ Required modules: - Module 01-08: Core MLP functionality (from MNIST test) - Module 09: Convolutions (Conv2d, MaxPool2d) - Module 10: DataLoader for efficient batch processing -- Module 11: CNN training capabilities +- Module 08: Training capabilities for CNN This demonstrates the milestone: "Can train CNNs on CIFAR-10" """ diff --git a/tinytorch/tests/progressive/README.md b/tinytorch/tests/progressive/README.md index 2c16a41cc5..614e419128 100644 --- a/tinytorch/tests/progressive/README.md +++ b/tinytorch/tests/progressive/README.md @@ -14,12 +14,13 @@ Module 01: Tensor ← Foundation: if this breaks, everything breaks Module 02: Activations ← Builds on Tensor Module 03: Layers ← Uses Tensor + Activations Module 04: Losses ← Uses Tensor + Layers -Module 05: Autograd ← Core: patches Tensor with gradient tracking +Module 05: DataLoader ← Data pipelines for training +Module 06: Autograd ← Core: patches Tensor with gradient tracking ...and so on ``` -When you're working on Module 05 (Autograd), a bug could: -- Break Autograd itself (Module 05 tests catch this) +When you're working on Module 06 (Autograd), a bug could: +- Break Autograd itself (Module 06 tests catch this) - Break Tensor operations (Module 01 regression tests catch this) - Break how Layers integrate with Autograd (integration tests catch this) @@ -103,7 +104,8 @@ tito module test 05 # 2. Module 02 regression tests (are Activations still OK?) # 3. Module 03 regression tests (are Layers still OK?) # 4. Module 04 regression tests (are Losses still OK?) -# 5. Module 05 capability tests (does Autograd work?) +# 5. Module 05 capability tests (does DataLoader work?) +# 6. Module 06 capability tests (does Autograd work?) # 6. Integration tests (do they all work together?) ``` diff --git a/tinytorch/tests/progressive/test_module_06_autograd.py b/tinytorch/tests/progressive/test_module_06_autograd.py index 3b3495748a..5ebd4f45d6 100644 --- a/tinytorch/tests/progressive/test_module_06_autograd.py +++ b/tinytorch/tests/progressive/test_module_06_autograd.py @@ -1,5 +1,5 @@ """ -Module 05: Autograd - Progressive Testing +Module 06: Autograd - Progressive Testing ========================================== 🎯 LEARNING OBJECTIVES: @@ -509,13 +509,13 @@ class TestCommonMistakes: if __name__ == "__main__": print("=" * 70) - print("Module 05: Autograd - Progressive Tests") + print("Module 06: Autograd - Progressive Tests") print("=" * 70) print() print("To run these tests:") - print(" pytest tests/progressive/test_module_05_autograd.py -v") + print(" pytest tests/progressive/test_module_06_autograd.py -v") print() print("Or via tito:") - print(" tito module test 05") + print(" tito module test 06") print() pytest.main([__file__, "-v"]) diff --git a/tinytorch/tests/regression/test_gradient_flow_fixes.py b/tinytorch/tests/regression/test_gradient_flow_fixes.py index b335e445ec..cd6d9e39de 100644 --- a/tinytorch/tests/regression/test_gradient_flow_fixes.py +++ b/tinytorch/tests/regression/test_gradient_flow_fixes.py @@ -78,9 +78,9 @@ def test_regression_subtraction_has_backward(): """ Regression test for Issue #3: Subtraction had no backward pass. - Bug: Tensor.__sub__ not patched by Module 05, no gradient flow. - Fix: Added SubBackward class and patched __sub__ in Module 05. - Commit: Module 05 fixes + Bug: Tensor.__sub__ not patched by Module 06, no gradient flow. + Fix: Added SubBackward class and patched __sub__ in Module 06. + Commit: Module 06 fixes """ print("Testing regression: subtraction backward...") @@ -105,9 +105,9 @@ def test_regression_division_has_backward(): """ Regression test for Issue #4: Division had no backward pass. - Bug: Tensor.__truediv__ not patched by Module 05, no gradient flow. - Fix: Added DivBackward class and patched __truediv__ in Module 05. - Commit: Module 05 fixes + Bug: Tensor.__truediv__ not patched by Module 06, no gradient flow. + Fix: Added DivBackward class and patched __truediv__ in Module 06. + Commit: Module 06 fixes """ print("Testing regression: division backward...") @@ -213,9 +213,9 @@ def test_regression_transpose_has_backward(): """ Regression test for Issue #8: Transpose had no backward pass. - Bug: Tensor.transpose() not patched by Module 05, no gradient flow. - Fix: Added TransposeBackward class and patched transpose in Module 05. - Commit: Module 05 fixes (TransposeBackward) + Bug: Tensor.transpose() not patched by Module 06, no gradient flow. + Fix: Added TransposeBackward class and patched transpose in Module 06. + Commit: Module 06 fixes (TransposeBackward) """ print("Testing regression: transpose backward...") @@ -243,8 +243,8 @@ def test_regression_matmul_backward_uses_matmul(): Regression test for Issue #9: MatmulBackward used np.dot for gradients. Bug: MatmulBackward used np.dot which doesn't handle batched 3D+ tensors. - Fix: Changed to np.matmul and np.swapaxes in Module 05. - Commit: Module 05 fixes (MatmulBackward batched) + Fix: Changed to np.matmul and np.swapaxes in Module 06. + Commit: Module 06 fixes (MatmulBackward batched) """ print("Testing regression: MatmulBackward batched operations...") diff --git a/tinytorch/tests/run_training_milestone_tests.py b/tinytorch/tests/run_training_milestone_tests.py index dd7b6ac952..3ccf1a7e4c 100755 --- a/tinytorch/tests/run_training_milestone_tests.py +++ b/tinytorch/tests/run_training_milestone_tests.py @@ -8,9 +8,10 @@ Runs all tests for the core modules needed for the training milestone: 02_activations - Activation functions 03_layers - Neural network layers 04_losses - Loss functions - 05_autograd - Automatic differentiation - 06_optimizers - SGD, Adam, etc. - 07_training - Training loops + 05_dataloader - Data loading and batching + 06_autograd - Automatic differentiation + 07_optimizers - SGD, Adam, etc. + 08_training - Training loops This script runs all tests and provides a comprehensive summary. """ diff --git a/tinytorch/tinytorch/__init__.py b/tinytorch/tinytorch/__init__.py index 549f684191..c0e68d9a27 100644 --- a/tinytorch/tinytorch/__init__.py +++ b/tinytorch/tinytorch/__init__.py @@ -40,26 +40,7 @@ try: except ImportError: MSELoss = CrossEntropyLoss = BinaryCrossEntropyLoss = None -# Module 05: Autograd - Enable if available -try: - from .core.autograd import enable_autograd - enable_autograd() -except ImportError: - pass - -# Module 06: Optimizers -try: - from .core.optimizers import SGD, Adam, AdamW -except ImportError: - SGD = Adam = AdamW = None - -# Module 07: Training -try: - from .core.training import Trainer, CosineSchedule, clip_grad_norm -except ImportError: - Trainer = CosineSchedule = clip_grad_norm = None - -# Module 08: Data Loading +# Module 05: Data Loading try: from .core.dataloader import Dataset, TensorDataset, DataLoader from .core.dataloader import RandomHorizontalFlip, RandomCrop, Compose @@ -67,6 +48,25 @@ except ImportError: Dataset = TensorDataset = DataLoader = None RandomHorizontalFlip = RandomCrop = Compose = None +# Module 06: Autograd - Enable if available +try: + from .core.autograd import enable_autograd + enable_autograd() +except ImportError: + pass + +# Module 07: Optimizers +try: + from .core.optimizers import SGD, Adam, AdamW +except ImportError: + SGD = Adam = AdamW = None + +# Module 08: Training +try: + from .core.training import Trainer, CosineSchedule, clip_grad_norm +except ImportError: + Trainer = CosineSchedule = clip_grad_norm = None + # Module 09: Convolutions (CNN) try: from .core.spatial import Conv2d, MaxPool2d, AvgPool2d diff --git a/tinytorch/tito/commands/export_utils.py b/tinytorch/tito/commands/export_utils.py index 1135b24cd9..ac3e637435 100644 --- a/tinytorch/tito/commands/export_utils.py +++ b/tinytorch/tito/commands/export_utils.py @@ -20,10 +20,10 @@ SOURCE_MAPPINGS = { ("core", "activations"): "src/02_activations/02_activations.py", ("core", "layers"): "src/03_layers/03_layers.py", ("core", "losses"): "src/04_losses/04_losses.py", - ("core", "autograd"): "src/05_autograd/05_autograd.py", - ("core", "optimizers"): "src/06_optimizers/06_optimizers.py", - ("core", "training"): "src/07_training/07_training.py", - ("core", "dataloader"): "src/08_dataloader/08_dataloader.py", + ("core", "dataloader"): "src/05_dataloader/05_dataloader.py", + ("core", "autograd"): "src/06_autograd/06_autograd.py", + ("core", "optimizers"): "src/07_optimizers/07_optimizers.py", + ("core", "training"): "src/08_training/08_training.py", ("core", "convolutions"): "src/09_convolutions/09_convolutions.py", ("core", "tokenization"): "src/10_tokenization/10_tokenization.py", ("core", "embeddings"): "src/11_embeddings/11_embeddings.py", diff --git a/tinytorch/tito/core/status_analyzer.py b/tinytorch/tito/core/status_analyzer.py index f46a873225..e97507d4f1 100644 --- a/tinytorch/tito/core/status_analyzer.py +++ b/tinytorch/tito/core/status_analyzer.py @@ -232,13 +232,12 @@ class TinyTorchStatusAnalyzer: '15_mlops': 'ProductionMLOpsProfiler', '16_capstone': 'ProductionMLSystemProfiler', '07_attention': 'AttentionEfficiencyProfiler', - '09_autograd': 'AutogradSystemsProfiler', - '11_training': 'TrainingPipelineProfiler', - '03_activations': 'ActivationProfiler', - '04_layers': 'LayerArchitectureProfiler', - '05_dense': 'NetworkStabilityMonitor', - '09_convolutions': 'ConvolutionProfiler', - '08_dataloader': 'DataPipelineProfiler' + '06_autograd': 'AutogradSystemsProfiler', + '08_training': 'TrainingPipelineProfiler', + '02_activations': 'ActivationProfiler', + '03_layers': 'LayerArchitectureProfiler', + '05_dataloader': 'DataPipelineProfiler', + '09_convolutions': 'ConvolutionProfiler' } if module_name in required_profilers: