mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-07-22 05:06:53 -05:00
Vijay Janapa Reddi
763cdd2bf2
WHAT: Added Tensor.__getitem__ (slicing) following progressive disclosure principles MODULE 01 (Tensor): - Added __getitem__ method for basic slicing operations - Clean implementation with NO gradient mentions (progressive disclosure) - Supports all NumPy-style indexing: x[0], x[:3], x[1:4], x[:, 1] - Ensures scalar results are wrapped in arrays MODULE 05 (Autograd): - Added SliceBackward function for gradient computation - Implements proper gradient scatter: zeros everywhere except sliced positions - Added monkey-patching in enable_autograd() for __getitem__ - Follows same pattern as existing operations (add, mul, matmul) MODULE 11 (Embeddings): - Updated PositionalEncoding to use Tensor slicing instead of .data - Fixed multiple .data accesses that broke computation graphs - Removed Tensor() wrapping that created gradient-disconnected leafs - Uses proper Tensor operations to preserve gradient flow TESTING: - All 6 component tests PASS (Embedding, Attention, FFN, Residual, Forward, Training) - 19/19 parameters get gradients (was 18/19 before) - Loss dropping better: 1.54→1.08 (vs 1.62→1.24 before) - Model still not learning (0% accuracy) - needs fresh session to test monkey-patching WHY THIS MATTERS: - Tensor slicing is FUNDAMENTAL - needed by transformers for position embeddings - Progressive disclosure maintains educational integrity - Follows existing TinyTorch architecture patterns - Enables position embeddings to potentially learn (pending verification) DOCUMENTS CREATED: - milestones/05_2017_transformer/TENSOR_SLICING_IMPLEMENTATION.md - milestones/05_2017_transformer/STATUS.md - milestones/05_2017_transformer/FIXES_SUMMARY.md - milestones/05_2017_transformer/DEBUG_REVERSAL.md - tests/milestones/test_reversal_debug.py (component tests) ARCHITECTURAL PRINCIPLE: Progressive disclosure is not just nice-to-have, it's CRITICAL for educational systems. Don't expose Module 05 concepts (gradients) in Module 01 (basic operations). Monkey-patch when features are needed, not before.
100 lines
4.3 KiB
Python
Generated
100 lines
4.3 KiB
Python
Generated
# ╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
# ║ 🚨 CRITICAL WARNING 🚨 ║
|
|
# ║ AUTOGENERATED! DO NOT EDIT! ║
|
|
# ║ ║
|
|
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
|
|
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
|
|
# ║ ║
|
|
# ║ ✅ TO EDIT: modules/XX_compression/compression.py ║
|
|
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
|
|
# ║ ║
|
|
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
|
|
# ║ Editing it directly may break module functionality and training. ║
|
|
# ║ ║
|
|
# ║ 🎓 LEARNING TIP: Work in modules/ - that's where real development ║
|
|
# ║ happens! The tinytorch/ directory is just the compiled output. ║
|
|
# ╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
# %% auto 0
|
|
__all__ = ['Tensor', 'Linear', 'Sequential']
|
|
|
|
# %% ../../modules/source/17_compression/compression_dev.ipynb 1
|
|
import numpy as np
|
|
import copy
|
|
from typing import List, Dict, Any, Tuple, Optional
|
|
import time
|
|
|
|
# Import from previous modules
|
|
# Note: In the full package, these would be imports like:
|
|
# from tinytorch.core.tensor import Tensor
|
|
# from tinytorch.core.layers import Linear
|
|
# For development, we'll create minimal implementations
|
|
|
|
class Tensor:
|
|
"""Minimal Tensor class for compression development - imports from Module 01 in practice."""
|
|
def __init__(self, data, requires_grad=False):
|
|
self.data = np.array(data)
|
|
self.shape = self.data.shape
|
|
self.size = self.data.size
|
|
self.requires_grad = requires_grad
|
|
self.grad = None
|
|
|
|
def __add__(self, other):
|
|
if isinstance(other, Tensor):
|
|
return Tensor(self.data + other.data)
|
|
return Tensor(self.data + other)
|
|
|
|
def __mul__(self, other):
|
|
if isinstance(other, Tensor):
|
|
return Tensor(self.data * other.data)
|
|
return Tensor(self.data * other)
|
|
|
|
def matmul(self, other):
|
|
return Tensor(np.dot(self.data, other.data))
|
|
|
|
def abs(self):
|
|
return Tensor(np.abs(self.data))
|
|
|
|
def sum(self, axis=None):
|
|
return Tensor(self.data.sum(axis=axis))
|
|
|
|
def __repr__(self):
|
|
return f"Tensor(shape={self.shape})"
|
|
|
|
class Linear:
|
|
"""Minimal Linear layer for compression development - imports from Module 03 in practice."""
|
|
def __init__(self, in_features, out_features, bias=True):
|
|
self.in_features = in_features
|
|
self.out_features = out_features
|
|
# Initialize with He initialization
|
|
self.weight = Tensor(np.random.randn(in_features, out_features) * np.sqrt(2.0 / in_features))
|
|
self.bias = Tensor(np.zeros(out_features)) if bias else None
|
|
|
|
def forward(self, x):
|
|
output = x.matmul(self.weight)
|
|
if self.bias is not None:
|
|
output = output + self.bias
|
|
return output
|
|
|
|
def parameters(self):
|
|
params = [self.weight]
|
|
if self.bias is not None:
|
|
params.append(self.bias)
|
|
return params
|
|
|
|
class Sequential:
|
|
"""Minimal Sequential container for model compression."""
|
|
def __init__(self, *layers):
|
|
self.layers = list(layers)
|
|
|
|
def forward(self, x):
|
|
for layer in self.layers:
|
|
x = layer.forward(x)
|
|
return x
|
|
|
|
def parameters(self):
|
|
params = []
|
|
for layer in self.layers:
|
|
if hasattr(layer, 'parameters'):
|
|
params.extend(layer.parameters())
|
|
return params
|