github-starred/TinyTorch
2
0
Fork 0
mirror of https://github.com/MLSysBook/TinyTorch.git synced 2026-04-30 19:58:04 -05:00
Code Issues 7 Packages Projects Releases Wiki Activity

Add exported package files and cleanup

Browse Source 
This commit includes:
- Exported tinytorch package files from nbdev (autograd, losses, optimizers, training, etc.)
- Updated activations.py and layers.py with __call__ methods
- New module exports: attention, spatial, tokenization, transformer, etc.
- Removed old _modidx.py file
- Cleanup of duplicate milestone directories

These are the generated package files that correspond to the source modules
we've been developing. Students will import from these when using TinyTorch.
This commit is contained in:
Vijay Janapa Reddi
2025-09-30 12:38:56 -04:00
parent 2377c788fe
commit 1f23035a1e
24 changed files with 478 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
tinytorch/_modidx.py generated
View File

@@ -1,109 +0,0 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/[unknown]/[unknown]_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# Autogenerated by nbdev
d = { 'settings': { 'branch': 'main',
'doc_baseurl': '/TinyTorch/',
'doc_host': 'https://tinytorch.github.io',
'git_url': 'https://github.com/tinytorch/TinyTorch/',
'lib_path': 'tinytorch'},
'syms': { 'tinytorch.benchmarking.benchmark': {},
'tinytorch.core.activations': { 'tinytorch.core.activations.GELU': ( '02_activations/activations_dev.html#gelu',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.GELU.backward': ( '02_activations/activations_dev.html#gelu.backward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.GELU.forward': ( '02_activations/activations_dev.html#gelu.forward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.ReLU': ( '02_activations/activations_dev.html#relu',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.ReLU.backward': ( '02_activations/activations_dev.html#relu.backward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.ReLU.forward': ( '02_activations/activations_dev.html#relu.forward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Sigmoid': ( '02_activations/activations_dev.html#sigmoid',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Sigmoid.__call__': ( '02_activations/activations_dev.html#sigmoid.__call__',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Sigmoid.backward': ( '02_activations/activations_dev.html#sigmoid.backward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Sigmoid.forward': ( '02_activations/activations_dev.html#sigmoid.forward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Softmax': ( '02_activations/activations_dev.html#softmax',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Softmax.backward': ( '02_activations/activations_dev.html#softmax.backward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Softmax.forward': ( '02_activations/activations_dev.html#softmax.forward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Tanh': ( '02_activations/activations_dev.html#tanh',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Tanh.backward': ( '02_activations/activations_dev.html#tanh.backward',
'tinytorch/core/activations.py'),
'tinytorch.core.activations.Tanh.forward': ( '02_activations/activations_dev.html#tanh.forward',
'tinytorch/core/activations.py')},
'tinytorch.core.layers': { 'tinytorch.core.layers.Dropout': ('03_layers/layers_dev.html#dropout', 'tinytorch/core/layers.py'),
'tinytorch.core.layers.Dropout.__init__': ( '03_layers/layers_dev.html#dropout.__init__',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Dropout.__repr__': ( '03_layers/layers_dev.html#dropout.__repr__',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Dropout.forward': ( '03_layers/layers_dev.html#dropout.forward',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Dropout.parameters': ( '03_layers/layers_dev.html#dropout.parameters',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear': ('03_layers/layers_dev.html#linear', 'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear.__call__': ( '03_layers/layers_dev.html#linear.__call__',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear.__init__': ( '03_layers/layers_dev.html#linear.__init__',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear.__repr__': ( '03_layers/layers_dev.html#linear.__repr__',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear.forward': ( '03_layers/layers_dev.html#linear.forward',
'tinytorch/core/layers.py'),
'tinytorch.core.layers.Linear.parameters': ( '03_layers/layers_dev.html#linear.parameters',
'tinytorch/core/layers.py')},
'tinytorch.core.tensor': { 'tinytorch.core.tensor.Tensor': ('01_tensor/tensor_dev.html#tensor', 'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__add__': ( '01_tensor/tensor_dev.html#tensor.__add__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__init__': ( '01_tensor/tensor_dev.html#tensor.__init__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__mul__': ( '01_tensor/tensor_dev.html#tensor.__mul__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__repr__': ( '01_tensor/tensor_dev.html#tensor.__repr__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__str__': ( '01_tensor/tensor_dev.html#tensor.__str__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__sub__': ( '01_tensor/tensor_dev.html#tensor.__sub__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.__truediv__': ( '01_tensor/tensor_dev.html#tensor.__truediv__',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.backward': ( '01_tensor/tensor_dev.html#tensor.backward',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.matmul': ( '01_tensor/tensor_dev.html#tensor.matmul',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.max': ( '01_tensor/tensor_dev.html#tensor.max',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.mean': ( '01_tensor/tensor_dev.html#tensor.mean',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.numpy': ( '01_tensor/tensor_dev.html#tensor.numpy',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.reshape': ( '01_tensor/tensor_dev.html#tensor.reshape',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.sum': ( '01_tensor/tensor_dev.html#tensor.sum',
'tinytorch/core/tensor.py'),
'tinytorch.core.tensor.Tensor.transpose': ( '01_tensor/tensor_dev.html#tensor.transpose',
'tinytorch/core/tensor.py')},
'tinytorch.data.loader': {},
'tinytorch.profiling.profiler': {}}}

0
tinytorch/applications/__init__.py generated Normal file
View File

8
tinytorch/applications/tinygpt.py generated Normal file
View File

@@ -0,0 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/20_capstone/capstone_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/20_capstone/capstone_dev.ipynb 2
#| default_exp applications.tinygpt
#| export

18
tinytorch/benchmarking/benchmark.py generated
View File

@@ -1,19 +1,5 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/XX_benchmark/benchmark_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/19_benchmarking/benchmarking_dev.ipynb.
# %% auto 0
__all__ = []

18
tinytorch/core/activations.py generated
View File

@@ -1,19 +1,5 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/03_activations/activations_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/02_activations/activations_dev.ipynb.
# %% auto 0
__all__ = ['Sigmoid', 'ReLU', 'Tanh', 'GELU', 'Softmax']

8
tinytorch/core/attention.py generated Normal file
View File

@@ -0,0 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/12_attention/attention_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/12_attention/attention_dev.ipynb 0
#| default_exp core.attention
#| export

12
tinytorch/core/autograd.py generated Normal file
View File

@@ -0,0 +1,12 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/05_autograd/autograd_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/05_autograd/autograd_dev.ipynb 1
import numpy as np
from typing import Optional, List, Tuple
import sys
import os
from .tensor import Tensor

18
tinytorch/core/layers.py generated
View File

@@ -1,19 +1,5 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/04_layers/layers_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/03_layers/layers_dev.ipynb.
# %% auto 0
__all__ = ['Linear', 'Dropout']

22
tinytorch/core/losses.py generated Normal file
View File

@@ -0,0 +1,22 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/04_losses/losses_dev.ipynb.
# %% auto 0
__all__ = ['import_previous_module']
# %% ../../modules/source/04_losses/losses_dev.ipynb 3
import numpy as np
import matplotlib.pyplot as plt
import time
from typing import Optional
def import_previous_module(module_name: str, component_name: str):
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..', module_name))
module = __import__(f"{module_name.split('_')[1]}_dev")
return getattr(module, component_name)
# Import from tinytorch package
from .tensor import Tensor
from .layers import Linear
from .activations import ReLU

14
tinytorch/core/optimizers.py generated Normal file
View File

@@ -0,0 +1,14 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/06_optimizers/optimizers_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/06_optimizers/optimizers_dev.ipynb 1
import numpy as np
from typing import List, Union, Optional, Dict, Any
# Import Tensor from Module 01 (now with gradient support from Module 05)
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor'))
from tensor_dev import Tensor

64
tinytorch/core/spatial.py generated Normal file
View File

@@ -0,0 +1,64 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/09_spatial/spatial_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/09_spatial/spatial_dev.ipynb 1
import numpy as np
import sys
import os
import time
# Import dependencies from other modules
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor'))
from tensor_dev import Tensor
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '03_layers'))
from layers_dev import Module
# Note: Keeping simplified implementations for reference during development
class _SimplifiedTensor:
"""Simplified tensor for spatial operations development."""
def __init__(self, data, requires_grad=False):
self.data = np.array(data, dtype=np.float32)
self.shape = self.data.shape
self.requires_grad = requires_grad
self.grad = None
def __repr__(self):
return f"Tensor(shape={self.shape}, data=\n{self.data})"
def __add__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data + other.data)
return Tensor(self.data + other)
def __mul__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data * other.data)
return Tensor(self.data * other)
def sum(self):
return Tensor(np.sum(self.data))
def mean(self):
return Tensor(np.mean(self.data))
# Create a simple Module base class for inheritance
class Module:
"""Simple base class for neural network modules."""
def __init__(self):
pass
def forward(self, x):
raise NotImplementedError("Subclasses must implement forward()")
def parameters(self):
"""Return list of parameters for this module."""
params = []
for attr_name in dir(self):
attr = getattr(self, attr_name)
if hasattr(attr, 'data') and hasattr(attr, 'requires_grad'):
params.append(attr)
return params

18
tinytorch/core/tensor.py generated
View File

@@ -1,19 +1,5 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/02_tensor/tensor_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/01_tensor/tensor_dev.ipynb.
# %% auto 0
__all__ = ['Tensor']

26
tinytorch/core/training.py generated Normal file
View File

@@ -0,0 +1,26 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/07_training/training_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/07_training/training_dev.ipynb 1
import numpy as np
import pickle
import time
from typing import Dict, List, Optional, Tuple, Any, Callable
from pathlib import Path
import sys
import os
# Import dependencies from other modules
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor'))
from tensor_dev import Tensor
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '03_layers'))
from layers_dev import Linear
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '04_losses'))
from losses_dev import MSELoss, CrossEntropyLoss
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '06_optimizers'))
from optimizers_dev import SGD, AdamW

0
tinytorch/generation/__init__.py generated Normal file
View File

57
tinytorch/generation/kv_cache.py generated Normal file
View File

@@ -0,0 +1,57 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/14_kvcaching/kvcaching_dev.ipynb.
# %% auto 0
__all__ = ['Tensor']
# %% ../../modules/source/14_kvcaching/kvcaching_dev.ipynb 1
import numpy as np
import time
from typing import Tuple, Optional, Dict, List
from dataclasses import dataclass
# Import our TinyTorch components (Modules 01-13)
### BEGIN SOLUTION
# Note: In real implementation, these would import from previous modules
# For now, we'll implement minimal versions to focus on caching concepts
class Tensor:
"""Minimal Tensor for KV Caching focus (from Module 01)"""
def __init__(self, data, requires_grad=False):
self.data = np.array(data)
self.shape = self.data.shape
self.requires_grad = requires_grad
self.grad = None
def __getitem__(self, key):
return Tensor(self.data[key])
def __setitem__(self, key, value):
if isinstance(value, Tensor):
self.data[key] = value.data
else:
self.data[key] = value
def size(self, dim=None):
if dim is None:
return self.shape
return self.shape[dim]
def view(self, *shape):
return Tensor(self.data.reshape(shape))
def transpose(self, dim0, dim1):
axes = list(range(len(self.shape)))
axes[dim0], axes[dim1] = axes[dim1], axes[dim0]
return Tensor(np.transpose(self.data, axes))
@staticmethod
def cat(tensors, dim=0):
"""Concatenate tensors along dimension"""
arrays = [t.data for t in tensors]
return Tensor(np.concatenate(arrays, axis=dim))
@staticmethod
def zeros(*shape):
"""Create zero tensor"""
return Tensor(np.zeros(shape))
### END SOLUTION

0
tinytorch/models/__init__.py generated Normal file
View File

148
tinytorch/models/transformer.py generated Normal file
View File

@@ -0,0 +1,148 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/13_transformers/transformers_dev.ipynb.
# %% auto 0
__all__ = ['Tensor', 'Linear', 'MultiHeadAttention', 'Embedding', 'gelu']
# %% ../../modules/source/13_transformers/transformers_dev.ipynb 1
import numpy as np
import math
from typing import Optional, List
# Minimal implementations for development - in practice these import from previous modules
class Tensor:
"""Minimal Tensor class for transformer development - imports from Module 01 in practice."""
def __init__(self, data, requires_grad=False):
self.data = np.array(data)
self.shape = self.data.shape
self.size = self.data.size
self.requires_grad = requires_grad
self.grad = None
def __add__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data + other.data)
return Tensor(self.data + other)
def __mul__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data * other.data)
return Tensor(self.data * other)
def matmul(self, other):
return Tensor(np.dot(self.data, other.data))
def sum(self, axis=None, keepdims=False):
return Tensor(self.data.sum(axis=axis, keepdims=keepdims))
def mean(self, axis=None, keepdims=False):
return Tensor(self.data.mean(axis=axis, keepdims=keepdims))
def reshape(self, *shape):
return Tensor(self.data.reshape(shape))
def __repr__(self):
return f"Tensor(data={self.data}, shape={self.shape})"
class Linear:
"""Minimal Linear layer - imports from Module 03 in practice."""
def __init__(self, in_features, out_features, bias=True):
# Xavier/Glorot initialization
std = math.sqrt(2.0 / (in_features + out_features))
self.weight = Tensor(np.random.normal(0, std, (in_features, out_features)))
self.bias = Tensor(np.zeros(out_features)) if bias else None
def forward(self, x):
output = x.matmul(self.weight)
if self.bias is not None:
output = output + self.bias
return output
def parameters(self):
params = [self.weight]
if self.bias is not None:
params.append(self.bias)
return params
class MultiHeadAttention:
"""Minimal MultiHeadAttention - imports from Module 12 in practice."""
def __init__(self, embed_dim, num_heads):
assert embed_dim % num_heads == 0
self.embed_dim = embed_dim
self.num_heads = num_heads
self.head_dim = embed_dim // num_heads
self.q_proj = Linear(embed_dim, embed_dim)
self.k_proj = Linear(embed_dim, embed_dim)
self.v_proj = Linear(embed_dim, embed_dim)
self.out_proj = Linear(embed_dim, embed_dim)
def forward(self, x, mask=None):
batch_size, seq_len, embed_dim = x.shape
# Linear projections
Q = self.q_proj.forward(x)
K = self.k_proj.forward(x)
V = self.v_proj.forward(x)
# Reshape for multi-head attention
Q = Q.reshape(batch_size, seq_len, self.num_heads, self.head_dim)
K = K.reshape(batch_size, seq_len, self.num_heads, self.head_dim)
V = V.reshape(batch_size, seq_len, self.num_heads, self.head_dim)
# Transpose to (batch_size, num_heads, seq_len, head_dim)
Q = Tensor(np.transpose(Q.data, (0, 2, 1, 3)))
K = Tensor(np.transpose(K.data, (0, 2, 1, 3)))
V = Tensor(np.transpose(V.data, (0, 2, 1, 3)))
# Scaled dot-product attention
scores = Tensor(np.matmul(Q.data, np.transpose(K.data, (0, 1, 3, 2))))
scores = scores * (1.0 / math.sqrt(self.head_dim))
# Apply causal mask for autoregressive generation
if mask is not None:
scores = Tensor(scores.data + mask.data)
# Softmax
attention_weights = self._softmax(scores)
# Apply attention to values
out = Tensor(np.matmul(attention_weights.data, V.data))
# Transpose back and reshape
out = Tensor(np.transpose(out.data, (0, 2, 1, 3)))
out = out.reshape(batch_size, seq_len, embed_dim)
# Final linear projection
return self.out_proj.forward(out)
def _softmax(self, x):
"""Numerically stable softmax."""
exp_x = Tensor(np.exp(x.data - np.max(x.data, axis=-1, keepdims=True)))
return Tensor(exp_x.data / np.sum(exp_x.data, axis=-1, keepdims=True))
def parameters(self):
params = []
params.extend(self.q_proj.parameters())
params.extend(self.k_proj.parameters())
params.extend(self.v_proj.parameters())
params.extend(self.out_proj.parameters())
return params
class Embedding:
"""Minimal Embedding layer - imports from Module 11 in practice."""
def __init__(self, vocab_size, embed_dim):
self.vocab_size = vocab_size
self.embed_dim = embed_dim
# Initialize with small random values
self.weight = Tensor(np.random.normal(0, 0.02, (vocab_size, embed_dim)))
def forward(self, indices):
# Simple embedding lookup
return Tensor(self.weight.data[indices.data])
def parameters(self):
return [self.weight]
def gelu(x):
"""GELU activation function."""
return Tensor(0.5 * x.data * (1 + np.tanh(np.sqrt(2 / np.pi) * (x.data + 0.044715 * x.data**3))))

0
tinytorch/optimization/__init__.py generated Normal file
View File

8
tinytorch/optimization/acceleration.py generated Normal file
View File

@@ -0,0 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/16_acceleration/acceleration_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/16_acceleration/acceleration_dev.ipynb 0
#| default_exp optimization.acceleration
#| export

85
tinytorch/optimization/compression.py generated Normal file
View File

@@ -0,0 +1,85 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/18_compression/compression_dev.ipynb.
# %% auto 0
__all__ = ['Tensor', 'Linear', 'Sequential']
# %% ../../modules/source/18_compression/compression_dev.ipynb 1
import numpy as np
import copy
from typing import List, Dict, Any, Tuple, Optional
import time
# Import from previous modules
# Note: In the full package, these would be imports like:
# from tinytorch.core.tensor import Tensor
# from tinytorch.core.layers import Linear
# For development, we'll create minimal implementations
class Tensor:
"""Minimal Tensor class for compression development - imports from Module 01 in practice."""
def __init__(self, data, requires_grad=False):
self.data = np.array(data)
self.shape = self.data.shape
self.size = self.data.size
self.requires_grad = requires_grad
self.grad = None
def __add__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data + other.data)
return Tensor(self.data + other)
def __mul__(self, other):
if isinstance(other, Tensor):
return Tensor(self.data * other.data)
return Tensor(self.data * other)
def matmul(self, other):
return Tensor(np.dot(self.data, other.data))
def abs(self):
return Tensor(np.abs(self.data))
def sum(self, axis=None):
return Tensor(self.data.sum(axis=axis))
def __repr__(self):
return f"Tensor(shape={self.shape})"
class Linear:
"""Minimal Linear layer for compression development - imports from Module 03 in practice."""
def __init__(self, in_features, out_features, bias=True):
self.in_features = in_features
self.out_features = out_features
# Initialize with He initialization
self.weight = Tensor(np.random.randn(in_features, out_features) * np.sqrt(2.0 / in_features))
self.bias = Tensor(np.zeros(out_features)) if bias else None
def forward(self, x):
output = x.matmul(self.weight)
if self.bias is not None:
output = output + self.bias
return output
def parameters(self):
params = [self.weight]
if self.bias is not None:
params.append(self.bias)
return params
class Sequential:
"""Minimal Sequential container for model compression."""
def __init__(self, *layers):
self.layers = list(layers)
def forward(self, x):
for layer in self.layers:
x = layer.forward(x)
return x
def parameters(self):
params = []
for layer in self.layers:
if hasattr(layer, 'parameters'):
params.extend(layer.parameters())
return params

8
tinytorch/optimization/quantization.py generated Normal file
View File

@@ -0,0 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/17_quantization/quantization_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/17_quantization/quantization_dev.ipynb 0
#| default_exp optimization.quantization
#| export

18
tinytorch/profiling/profiler.py generated
View File

@@ -1,19 +1,5 @@
# ╔═══════════════════════════════════════════════════════════════════════════════╗
# ║ 🚨 CRITICAL WARNING 🚨 ║
# ║ AUTOGENERATED! DO NOT EDIT! ║
# ║ ║
# ║ This file is AUTOMATICALLY GENERATED from source modules. ║
# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║
# ║ ║
# ║ ✅ TO EDIT: modules/source/XX_profiler/profiler_dev.py ║
# ║ ✅ TO EXPORT: Run 'tito module complete <module_name>' ║
# ║ ║
# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║
# ║ Editing it directly may break module functionality and training. ║
# ║ ║
# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║
# ║ happens! The tinytorch/ directory is just the compiled output. ║
# ╚═══════════════════════════════════════════════════════════════════════════════╝
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/15_profiling/profiling_dev.ipynb.
# %% auto 0
__all__ = []

0
tinytorch/text/__init__.py generated Normal file
View File

8
tinytorch/text/tokenization.py generated Normal file
View File

@@ -0,0 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/10_tokenization/tokenization_dev.ipynb.
# %% auto 0
__all__ = []
# %% ../../modules/source/10_tokenization/tokenization_dev.ipynb 0
#| default_exp text.tokenization
#| export