diff --git a/tinytorch/_modidx.py b/tinytorch/_modidx.py deleted file mode 100644 index 2048c8af..00000000 --- a/tinytorch/_modidx.py +++ /dev/null @@ -1,109 +0,0 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/[unknown]/[unknown]_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ -# Autogenerated by nbdev - -d = { 'settings': { 'branch': 'main', - 'doc_baseurl': '/TinyTorch/', - 'doc_host': 'https://tinytorch.github.io', - 'git_url': 'https://github.com/tinytorch/TinyTorch/', - 'lib_path': 'tinytorch'}, - 'syms': { 'tinytorch.benchmarking.benchmark': {}, - 'tinytorch.core.activations': { 'tinytorch.core.activations.GELU': ( '02_activations/activations_dev.html#gelu', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.GELU.backward': ( '02_activations/activations_dev.html#gelu.backward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.GELU.forward': ( '02_activations/activations_dev.html#gelu.forward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.ReLU': ( '02_activations/activations_dev.html#relu', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.ReLU.backward': ( '02_activations/activations_dev.html#relu.backward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.ReLU.forward': ( '02_activations/activations_dev.html#relu.forward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Sigmoid': ( '02_activations/activations_dev.html#sigmoid', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Sigmoid.__call__': ( '02_activations/activations_dev.html#sigmoid.__call__', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Sigmoid.backward': ( '02_activations/activations_dev.html#sigmoid.backward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Sigmoid.forward': ( '02_activations/activations_dev.html#sigmoid.forward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Softmax': ( '02_activations/activations_dev.html#softmax', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Softmax.backward': ( '02_activations/activations_dev.html#softmax.backward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Softmax.forward': ( '02_activations/activations_dev.html#softmax.forward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Tanh': ( '02_activations/activations_dev.html#tanh', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Tanh.backward': ( '02_activations/activations_dev.html#tanh.backward', - 'tinytorch/core/activations.py'), - 'tinytorch.core.activations.Tanh.forward': ( '02_activations/activations_dev.html#tanh.forward', - 'tinytorch/core/activations.py')}, - 'tinytorch.core.layers': { 'tinytorch.core.layers.Dropout': ('03_layers/layers_dev.html#dropout', 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Dropout.__init__': ( '03_layers/layers_dev.html#dropout.__init__', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Dropout.__repr__': ( '03_layers/layers_dev.html#dropout.__repr__', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Dropout.forward': ( '03_layers/layers_dev.html#dropout.forward', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Dropout.parameters': ( '03_layers/layers_dev.html#dropout.parameters', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear': ('03_layers/layers_dev.html#linear', 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear.__call__': ( '03_layers/layers_dev.html#linear.__call__', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear.__init__': ( '03_layers/layers_dev.html#linear.__init__', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear.__repr__': ( '03_layers/layers_dev.html#linear.__repr__', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear.forward': ( '03_layers/layers_dev.html#linear.forward', - 'tinytorch/core/layers.py'), - 'tinytorch.core.layers.Linear.parameters': ( '03_layers/layers_dev.html#linear.parameters', - 'tinytorch/core/layers.py')}, - 'tinytorch.core.tensor': { 'tinytorch.core.tensor.Tensor': ('01_tensor/tensor_dev.html#tensor', 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__add__': ( '01_tensor/tensor_dev.html#tensor.__add__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__init__': ( '01_tensor/tensor_dev.html#tensor.__init__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__mul__': ( '01_tensor/tensor_dev.html#tensor.__mul__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__repr__': ( '01_tensor/tensor_dev.html#tensor.__repr__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__str__': ( '01_tensor/tensor_dev.html#tensor.__str__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__sub__': ( '01_tensor/tensor_dev.html#tensor.__sub__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.__truediv__': ( '01_tensor/tensor_dev.html#tensor.__truediv__', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.backward': ( '01_tensor/tensor_dev.html#tensor.backward', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.matmul': ( '01_tensor/tensor_dev.html#tensor.matmul', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.max': ( '01_tensor/tensor_dev.html#tensor.max', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.mean': ( '01_tensor/tensor_dev.html#tensor.mean', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.numpy': ( '01_tensor/tensor_dev.html#tensor.numpy', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.reshape': ( '01_tensor/tensor_dev.html#tensor.reshape', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.sum': ( '01_tensor/tensor_dev.html#tensor.sum', - 'tinytorch/core/tensor.py'), - 'tinytorch.core.tensor.Tensor.transpose': ( '01_tensor/tensor_dev.html#tensor.transpose', - 'tinytorch/core/tensor.py')}, - 'tinytorch.data.loader': {}, - 'tinytorch.profiling.profiler': {}}} diff --git a/tinytorch/applications/__init__.py b/tinytorch/applications/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinytorch/applications/tinygpt.py b/tinytorch/applications/tinygpt.py new file mode 100644 index 00000000..80dabc9a --- /dev/null +++ b/tinytorch/applications/tinygpt.py @@ -0,0 +1,8 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/20_capstone/capstone_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/20_capstone/capstone_dev.ipynb 2 +#| default_exp applications.tinygpt +#| export diff --git a/tinytorch/benchmarking/benchmark.py b/tinytorch/benchmarking/benchmark.py index 8856ac98..59888381 100644 --- a/tinytorch/benchmarking/benchmark.py +++ b/tinytorch/benchmarking/benchmark.py @@ -1,19 +1,5 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/XX_benchmark/benchmark_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/19_benchmarking/benchmarking_dev.ipynb. + # %% auto 0 __all__ = [] diff --git a/tinytorch/core/activations.py b/tinytorch/core/activations.py index 7ea930a0..a61df44b 100644 --- a/tinytorch/core/activations.py +++ b/tinytorch/core/activations.py @@ -1,19 +1,5 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/03_activations/activations_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/02_activations/activations_dev.ipynb. + # %% auto 0 __all__ = ['Sigmoid', 'ReLU', 'Tanh', 'GELU', 'Softmax'] diff --git a/tinytorch/core/attention.py b/tinytorch/core/attention.py new file mode 100644 index 00000000..7d6df426 --- /dev/null +++ b/tinytorch/core/attention.py @@ -0,0 +1,8 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/12_attention/attention_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/12_attention/attention_dev.ipynb 0 +#| default_exp core.attention +#| export diff --git a/tinytorch/core/autograd.py b/tinytorch/core/autograd.py new file mode 100644 index 00000000..dcc565d8 --- /dev/null +++ b/tinytorch/core/autograd.py @@ -0,0 +1,12 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/05_autograd/autograd_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/05_autograd/autograd_dev.ipynb 1 +import numpy as np +from typing import Optional, List, Tuple +import sys +import os + +from .tensor import Tensor diff --git a/tinytorch/core/layers.py b/tinytorch/core/layers.py index 2f117f88..7ea028c4 100644 --- a/tinytorch/core/layers.py +++ b/tinytorch/core/layers.py @@ -1,19 +1,5 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/04_layers/layers_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/03_layers/layers_dev.ipynb. + # %% auto 0 __all__ = ['Linear', 'Dropout'] diff --git a/tinytorch/core/losses.py b/tinytorch/core/losses.py new file mode 100644 index 00000000..8e72375c --- /dev/null +++ b/tinytorch/core/losses.py @@ -0,0 +1,22 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/04_losses/losses_dev.ipynb. + +# %% auto 0 +__all__ = ['import_previous_module'] + +# %% ../../modules/source/04_losses/losses_dev.ipynb 3 +import numpy as np +import matplotlib.pyplot as plt +import time +from typing import Optional + +def import_previous_module(module_name: str, component_name: str): + import sys + import os + sys.path.append(os.path.join(os.path.dirname(__file__), '..', module_name)) + module = __import__(f"{module_name.split('_')[1]}_dev") + return getattr(module, component_name) + +# Import from tinytorch package +from .tensor import Tensor +from .layers import Linear +from .activations import ReLU diff --git a/tinytorch/core/optimizers.py b/tinytorch/core/optimizers.py new file mode 100644 index 00000000..1d39357d --- /dev/null +++ b/tinytorch/core/optimizers.py @@ -0,0 +1,14 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/06_optimizers/optimizers_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/06_optimizers/optimizers_dev.ipynb 1 +import numpy as np +from typing import List, Union, Optional, Dict, Any + +# Import Tensor from Module 01 (now with gradient support from Module 05) +import sys +import os +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor')) +from tensor_dev import Tensor diff --git a/tinytorch/core/spatial.py b/tinytorch/core/spatial.py new file mode 100644 index 00000000..faa47403 --- /dev/null +++ b/tinytorch/core/spatial.py @@ -0,0 +1,64 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/09_spatial/spatial_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/09_spatial/spatial_dev.ipynb 1 +import numpy as np +import sys +import os +import time + +# Import dependencies from other modules +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor')) +from tensor_dev import Tensor + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '03_layers')) +from layers_dev import Module + +# Note: Keeping simplified implementations for reference during development +class _SimplifiedTensor: + """Simplified tensor for spatial operations development.""" + + def __init__(self, data, requires_grad=False): + self.data = np.array(data, dtype=np.float32) + self.shape = self.data.shape + self.requires_grad = requires_grad + self.grad = None + + def __repr__(self): + return f"Tensor(shape={self.shape}, data=\n{self.data})" + + def __add__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data + other.data) + return Tensor(self.data + other) + + def __mul__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data * other.data) + return Tensor(self.data * other) + + def sum(self): + return Tensor(np.sum(self.data)) + + def mean(self): + return Tensor(np.mean(self.data)) + + # Create a simple Module base class for inheritance + class Module: + """Simple base class for neural network modules.""" + def __init__(self): + pass + + def forward(self, x): + raise NotImplementedError("Subclasses must implement forward()") + + def parameters(self): + """Return list of parameters for this module.""" + params = [] + for attr_name in dir(self): + attr = getattr(self, attr_name) + if hasattr(attr, 'data') and hasattr(attr, 'requires_grad'): + params.append(attr) + return params diff --git a/tinytorch/core/tensor.py b/tinytorch/core/tensor.py index fb786066..22b11dad 100644 --- a/tinytorch/core/tensor.py +++ b/tinytorch/core/tensor.py @@ -1,19 +1,5 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/02_tensor/tensor_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/01_tensor/tensor_dev.ipynb. + # %% auto 0 __all__ = ['Tensor'] diff --git a/tinytorch/core/training.py b/tinytorch/core/training.py new file mode 100644 index 00000000..54947841 --- /dev/null +++ b/tinytorch/core/training.py @@ -0,0 +1,26 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/07_training/training_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/07_training/training_dev.ipynb 1 +import numpy as np +import pickle +import time +from typing import Dict, List, Optional, Tuple, Any, Callable +from pathlib import Path +import sys +import os + +# Import dependencies from other modules +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor')) +from tensor_dev import Tensor + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '03_layers')) +from layers_dev import Linear + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '04_losses')) +from losses_dev import MSELoss, CrossEntropyLoss + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '06_optimizers')) +from optimizers_dev import SGD, AdamW diff --git a/tinytorch/generation/__init__.py b/tinytorch/generation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinytorch/generation/kv_cache.py b/tinytorch/generation/kv_cache.py new file mode 100644 index 00000000..ed623a38 --- /dev/null +++ b/tinytorch/generation/kv_cache.py @@ -0,0 +1,57 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/14_kvcaching/kvcaching_dev.ipynb. + +# %% auto 0 +__all__ = ['Tensor'] + +# %% ../../modules/source/14_kvcaching/kvcaching_dev.ipynb 1 +import numpy as np +import time +from typing import Tuple, Optional, Dict, List +from dataclasses import dataclass + +# Import our TinyTorch components (Modules 01-13) +### BEGIN SOLUTION +# Note: In real implementation, these would import from previous modules +# For now, we'll implement minimal versions to focus on caching concepts + +class Tensor: + """Minimal Tensor for KV Caching focus (from Module 01)""" + def __init__(self, data, requires_grad=False): + self.data = np.array(data) + self.shape = self.data.shape + self.requires_grad = requires_grad + self.grad = None + + def __getitem__(self, key): + return Tensor(self.data[key]) + + def __setitem__(self, key, value): + if isinstance(value, Tensor): + self.data[key] = value.data + else: + self.data[key] = value + + def size(self, dim=None): + if dim is None: + return self.shape + return self.shape[dim] + + def view(self, *shape): + return Tensor(self.data.reshape(shape)) + + def transpose(self, dim0, dim1): + axes = list(range(len(self.shape))) + axes[dim0], axes[dim1] = axes[dim1], axes[dim0] + return Tensor(np.transpose(self.data, axes)) + + @staticmethod + def cat(tensors, dim=0): + """Concatenate tensors along dimension""" + arrays = [t.data for t in tensors] + return Tensor(np.concatenate(arrays, axis=dim)) + + @staticmethod + def zeros(*shape): + """Create zero tensor""" + return Tensor(np.zeros(shape)) +### END SOLUTION diff --git a/tinytorch/models/__init__.py b/tinytorch/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinytorch/models/transformer.py b/tinytorch/models/transformer.py new file mode 100644 index 00000000..6e63d29b --- /dev/null +++ b/tinytorch/models/transformer.py @@ -0,0 +1,148 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/13_transformers/transformers_dev.ipynb. + +# %% auto 0 +__all__ = ['Tensor', 'Linear', 'MultiHeadAttention', 'Embedding', 'gelu'] + +# %% ../../modules/source/13_transformers/transformers_dev.ipynb 1 +import numpy as np +import math +from typing import Optional, List + +# Minimal implementations for development - in practice these import from previous modules +class Tensor: + """Minimal Tensor class for transformer development - imports from Module 01 in practice.""" + def __init__(self, data, requires_grad=False): + self.data = np.array(data) + self.shape = self.data.shape + self.size = self.data.size + self.requires_grad = requires_grad + self.grad = None + + def __add__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data + other.data) + return Tensor(self.data + other) + + def __mul__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data * other.data) + return Tensor(self.data * other) + + def matmul(self, other): + return Tensor(np.dot(self.data, other.data)) + + def sum(self, axis=None, keepdims=False): + return Tensor(self.data.sum(axis=axis, keepdims=keepdims)) + + def mean(self, axis=None, keepdims=False): + return Tensor(self.data.mean(axis=axis, keepdims=keepdims)) + + def reshape(self, *shape): + return Tensor(self.data.reshape(shape)) + + def __repr__(self): + return f"Tensor(data={self.data}, shape={self.shape})" + +class Linear: + """Minimal Linear layer - imports from Module 03 in practice.""" + def __init__(self, in_features, out_features, bias=True): + # Xavier/Glorot initialization + std = math.sqrt(2.0 / (in_features + out_features)) + self.weight = Tensor(np.random.normal(0, std, (in_features, out_features))) + self.bias = Tensor(np.zeros(out_features)) if bias else None + + def forward(self, x): + output = x.matmul(self.weight) + if self.bias is not None: + output = output + self.bias + return output + + def parameters(self): + params = [self.weight] + if self.bias is not None: + params.append(self.bias) + return params + +class MultiHeadAttention: + """Minimal MultiHeadAttention - imports from Module 12 in practice.""" + def __init__(self, embed_dim, num_heads): + assert embed_dim % num_heads == 0 + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + + self.q_proj = Linear(embed_dim, embed_dim) + self.k_proj = Linear(embed_dim, embed_dim) + self.v_proj = Linear(embed_dim, embed_dim) + self.out_proj = Linear(embed_dim, embed_dim) + + def forward(self, x, mask=None): + batch_size, seq_len, embed_dim = x.shape + + # Linear projections + Q = self.q_proj.forward(x) + K = self.k_proj.forward(x) + V = self.v_proj.forward(x) + + # Reshape for multi-head attention + Q = Q.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + K = K.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + V = V.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + # Transpose to (batch_size, num_heads, seq_len, head_dim) + Q = Tensor(np.transpose(Q.data, (0, 2, 1, 3))) + K = Tensor(np.transpose(K.data, (0, 2, 1, 3))) + V = Tensor(np.transpose(V.data, (0, 2, 1, 3))) + + # Scaled dot-product attention + scores = Tensor(np.matmul(Q.data, np.transpose(K.data, (0, 1, 3, 2)))) + scores = scores * (1.0 / math.sqrt(self.head_dim)) + + # Apply causal mask for autoregressive generation + if mask is not None: + scores = Tensor(scores.data + mask.data) + + # Softmax + attention_weights = self._softmax(scores) + + # Apply attention to values + out = Tensor(np.matmul(attention_weights.data, V.data)) + + # Transpose back and reshape + out = Tensor(np.transpose(out.data, (0, 2, 1, 3))) + out = out.reshape(batch_size, seq_len, embed_dim) + + # Final linear projection + return self.out_proj.forward(out) + + def _softmax(self, x): + """Numerically stable softmax.""" + exp_x = Tensor(np.exp(x.data - np.max(x.data, axis=-1, keepdims=True))) + return Tensor(exp_x.data / np.sum(exp_x.data, axis=-1, keepdims=True)) + + def parameters(self): + params = [] + params.extend(self.q_proj.parameters()) + params.extend(self.k_proj.parameters()) + params.extend(self.v_proj.parameters()) + params.extend(self.out_proj.parameters()) + return params + +class Embedding: + """Minimal Embedding layer - imports from Module 11 in practice.""" + def __init__(self, vocab_size, embed_dim): + self.vocab_size = vocab_size + self.embed_dim = embed_dim + # Initialize with small random values + self.weight = Tensor(np.random.normal(0, 0.02, (vocab_size, embed_dim))) + + def forward(self, indices): + # Simple embedding lookup + return Tensor(self.weight.data[indices.data]) + + def parameters(self): + return [self.weight] + +def gelu(x): + """GELU activation function.""" + return Tensor(0.5 * x.data * (1 + np.tanh(np.sqrt(2 / np.pi) * (x.data + 0.044715 * x.data**3)))) diff --git a/tinytorch/optimization/__init__.py b/tinytorch/optimization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinytorch/optimization/acceleration.py b/tinytorch/optimization/acceleration.py new file mode 100644 index 00000000..d0ca1d13 --- /dev/null +++ b/tinytorch/optimization/acceleration.py @@ -0,0 +1,8 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/16_acceleration/acceleration_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/16_acceleration/acceleration_dev.ipynb 0 +#| default_exp optimization.acceleration +#| export diff --git a/tinytorch/optimization/compression.py b/tinytorch/optimization/compression.py new file mode 100644 index 00000000..01ef28ae --- /dev/null +++ b/tinytorch/optimization/compression.py @@ -0,0 +1,85 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/18_compression/compression_dev.ipynb. + +# %% auto 0 +__all__ = ['Tensor', 'Linear', 'Sequential'] + +# %% ../../modules/source/18_compression/compression_dev.ipynb 1 +import numpy as np +import copy +from typing import List, Dict, Any, Tuple, Optional +import time + +# Import from previous modules +# Note: In the full package, these would be imports like: +# from tinytorch.core.tensor import Tensor +# from tinytorch.core.layers import Linear +# For development, we'll create minimal implementations + +class Tensor: + """Minimal Tensor class for compression development - imports from Module 01 in practice.""" + def __init__(self, data, requires_grad=False): + self.data = np.array(data) + self.shape = self.data.shape + self.size = self.data.size + self.requires_grad = requires_grad + self.grad = None + + def __add__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data + other.data) + return Tensor(self.data + other) + + def __mul__(self, other): + if isinstance(other, Tensor): + return Tensor(self.data * other.data) + return Tensor(self.data * other) + + def matmul(self, other): + return Tensor(np.dot(self.data, other.data)) + + def abs(self): + return Tensor(np.abs(self.data)) + + def sum(self, axis=None): + return Tensor(self.data.sum(axis=axis)) + + def __repr__(self): + return f"Tensor(shape={self.shape})" + +class Linear: + """Minimal Linear layer for compression development - imports from Module 03 in practice.""" + def __init__(self, in_features, out_features, bias=True): + self.in_features = in_features + self.out_features = out_features + # Initialize with He initialization + self.weight = Tensor(np.random.randn(in_features, out_features) * np.sqrt(2.0 / in_features)) + self.bias = Tensor(np.zeros(out_features)) if bias else None + + def forward(self, x): + output = x.matmul(self.weight) + if self.bias is not None: + output = output + self.bias + return output + + def parameters(self): + params = [self.weight] + if self.bias is not None: + params.append(self.bias) + return params + +class Sequential: + """Minimal Sequential container for model compression.""" + def __init__(self, *layers): + self.layers = list(layers) + + def forward(self, x): + for layer in self.layers: + x = layer.forward(x) + return x + + def parameters(self): + params = [] + for layer in self.layers: + if hasattr(layer, 'parameters'): + params.extend(layer.parameters()) + return params diff --git a/tinytorch/optimization/quantization.py b/tinytorch/optimization/quantization.py new file mode 100644 index 00000000..f5bc7cfd --- /dev/null +++ b/tinytorch/optimization/quantization.py @@ -0,0 +1,8 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/17_quantization/quantization_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/17_quantization/quantization_dev.ipynb 0 +#| default_exp optimization.quantization +#| export diff --git a/tinytorch/profiling/profiler.py b/tinytorch/profiling/profiler.py index f87c5764..82aca521 100644 --- a/tinytorch/profiling/profiler.py +++ b/tinytorch/profiling/profiler.py @@ -1,19 +1,5 @@ -# ╔═══════════════════════════════════════════════════════════════════════════════╗ -# ║ 🚨 CRITICAL WARNING 🚨 ║ -# ║ AUTOGENERATED! DO NOT EDIT! ║ -# ║ ║ -# ║ This file is AUTOMATICALLY GENERATED from source modules. ║ -# ║ ANY CHANGES MADE HERE WILL BE LOST when modules are re-exported! ║ -# ║ ║ -# ║ ✅ TO EDIT: modules/source/XX_profiler/profiler_dev.py ║ -# ║ ✅ TO EXPORT: Run 'tito module complete ' ║ -# ║ ║ -# ║ 🛡️ STUDENT PROTECTION: This file contains optimized implementations. ║ -# ║ Editing it directly may break module functionality and training. ║ -# ║ ║ -# ║ 🎓 LEARNING TIP: Work in modules/source/ - that's where real development ║ -# ║ happens! The tinytorch/ directory is just the compiled output. ║ -# ╚═══════════════════════════════════════════════════════════════════════════════╝ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/15_profiling/profiling_dev.ipynb. + # %% auto 0 __all__ = [] diff --git a/tinytorch/text/__init__.py b/tinytorch/text/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinytorch/text/tokenization.py b/tinytorch/text/tokenization.py new file mode 100644 index 00000000..10a40e63 --- /dev/null +++ b/tinytorch/text/tokenization.py @@ -0,0 +1,8 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../../modules/source/10_tokenization/tokenization_dev.ipynb. + +# %% auto 0 +__all__ = [] + +# %% ../../modules/source/10_tokenization/tokenization_dev.ipynb 0 +#| default_exp text.tokenization +#| export