mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-06-01 23:51:01 -05:00
b785b706f2
- Restored tools/py_to_notebook.py as a focused, standalone tool - Updated tito notebooks command to use subprocess to call the separate tool - Maintains clean separation of concerns: tito.py for CLI orchestration, py_to_notebook.py for conversion logic - Updated documentation to use 'tito notebooks' command instead of direct tool calls - Benefits: easier debugging, better maintainability, focused single-responsibility modules
491 lines
19 KiB
Plaintext
491 lines
19 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"jupyter:\n",
|
|
" jupytext:\n",
|
|
" text_representation:\n",
|
|
" extension: .py\n",
|
|
" format_name: percent\n",
|
|
" format_version: '1.3'\n",
|
|
" jupytext_version: 1.17.1\n",
|
|
"---\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"# Module 1: Tensor - Core Data Structure\n",
|
|
"\n",
|
|
"Welcome to the Tensor module! This is where TinyTorch really begins. You'll implement the fundamental data structure that powers all ML systems.\n",
|
|
"\n",
|
|
"## Learning Goals\n",
|
|
"- Understand tensors as N-dimensional arrays with ML-specific operations\n",
|
|
"- Implement a complete Tensor class with arithmetic operations\n",
|
|
"- Handle shape management, data types, and memory layout\n",
|
|
"- Build the foundation for neural networks and automatic differentiation\n",
|
|
"\n",
|
|
"## Module \u2192 Package Structure\n",
|
|
"**\ud83c\udf93 Teaching vs. \ud83d\udd27 Building**: \n",
|
|
"- **Learning side**: Work in `modules/tensor/tensor_dev.py` \n",
|
|
"- **Building side**: Exports to `tinytorch/core/tensor.py`\n",
|
|
"\n",
|
|
"This module builds the core data structure that all other TinyTorch components will use.\n",
|
|
"\"\"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| default_exp core.tensor\n",
|
|
"\n",
|
|
"# Setup and imports\n",
|
|
"import numpy as np\n",
|
|
"import sys\n",
|
|
"from typing import Union, List, Tuple, Optional, Any\n",
|
|
"\n",
|
|
"print(\"\ud83d\udd25 TinyTorch Tensor Module\")\n",
|
|
"print(f\"NumPy version: {np.__version__}\")\n",
|
|
"print(f\"Python version: {sys.version_info.major}.{sys.version_info.minor}\")\n",
|
|
"print(\"Ready to build tensors!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"## Step 1: What is a Tensor?\n",
|
|
"\n",
|
|
"A **tensor** is an N-dimensional array with ML-specific operations. Think of it as:\n",
|
|
"- **Scalar** (0D): A single number - `5.0`\n",
|
|
"- **Vector** (1D): A list of numbers - `[1, 2, 3]` \n",
|
|
"- **Matrix** (2D): A 2D array - `[[1, 2], [3, 4]]`\n",
|
|
"- **Higher dimensions**: 3D, 4D, etc. for images, video, batches\n",
|
|
"\n",
|
|
"**Why not just use NumPy?** We will use NumPy internally, but our Tensor class will add:\n",
|
|
"- ML-specific operations (later: gradients, GPU support)\n",
|
|
"- Consistent API for neural networks\n",
|
|
"- Type safety and error checking\n",
|
|
"- Integration with the rest of TinyTorch\n",
|
|
"\n",
|
|
"Let's start building!\n",
|
|
"\"\"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class Tensor:\n",
|
|
" \"\"\"\n",
|
|
" TinyTorch Tensor: N-dimensional array with ML operations.\n",
|
|
" \n",
|
|
" The fundamental data structure for all TinyTorch operations.\n",
|
|
" Wraps NumPy arrays with ML-specific functionality.\n",
|
|
" \n",
|
|
" TODO: Implement the core Tensor class with data handling and properties.\n",
|
|
" \"\"\"\n",
|
|
" \n",
|
|
" def __init__(self, data: Union[int, float, List, np.ndarray], dtype: Optional[str] = None):\n",
|
|
" \"\"\"\n",
|
|
" Create a new tensor from data.\n",
|
|
" \n",
|
|
" Args:\n",
|
|
" data: Input data (scalar, list, or numpy array)\n",
|
|
" dtype: Data type ('float32', 'int32', etc.). Defaults to auto-detect.\n",
|
|
" \n",
|
|
" TODO: Implement tensor creation with proper type handling.\n",
|
|
" \"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def data(self) -> np.ndarray:\n",
|
|
" \"\"\"Access underlying numpy array.\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def shape(self) -> Tuple[int, ...]:\n",
|
|
" \"\"\"Get tensor shape.\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def size(self) -> int:\n",
|
|
" \"\"\"Get total number of elements.\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def dtype(self) -> np.dtype:\n",
|
|
" \"\"\"Get data type as numpy dtype.\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" def __repr__(self) -> str:\n",
|
|
" \"\"\"String representation.\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#| export\n",
|
|
"class Tensor:\n",
|
|
" \"\"\"\n",
|
|
" TinyTorch Tensor: N-dimensional array with ML operations.\n",
|
|
" \n",
|
|
" The fundamental data structure for all TinyTorch operations.\n",
|
|
" Wraps NumPy arrays with ML-specific functionality.\n",
|
|
" \"\"\"\n",
|
|
" \n",
|
|
" def __init__(self, data: Union[int, float, List, np.ndarray], dtype: Optional[str] = None):\n",
|
|
" \"\"\"\n",
|
|
" Create a new tensor from data.\n",
|
|
" \n",
|
|
" Args:\n",
|
|
" data: Input data (scalar, list, or numpy array)\n",
|
|
" dtype: Data type ('float32', 'int32', etc.). Defaults to auto-detect.\n",
|
|
" \"\"\"\n",
|
|
" # Convert input to numpy array\n",
|
|
" if isinstance(data, (int, float, np.number)):\n",
|
|
" # Handle Python and NumPy scalars\n",
|
|
" if dtype is None:\n",
|
|
" # Auto-detect type: int for integers, float32 for floats\n",
|
|
" if isinstance(data, int) or (isinstance(data, np.number) and np.issubdtype(type(data), np.integer)):\n",
|
|
" dtype = 'int32'\n",
|
|
" else:\n",
|
|
" dtype = 'float32'\n",
|
|
" self._data = np.array(data, dtype=dtype)\n",
|
|
" elif isinstance(data, list):\n",
|
|
" # Let NumPy auto-detect type, then convert if needed\n",
|
|
" temp_array = np.array(data)\n",
|
|
" if dtype is None:\n",
|
|
" # Keep NumPy's auto-detected type, but prefer common ML types\n",
|
|
" if np.issubdtype(temp_array.dtype, np.integer):\n",
|
|
" dtype = 'int32'\n",
|
|
" elif np.issubdtype(temp_array.dtype, np.floating):\n",
|
|
" dtype = 'float32'\n",
|
|
" else:\n",
|
|
" dtype = temp_array.dtype\n",
|
|
" self._data = temp_array.astype(dtype)\n",
|
|
" elif isinstance(data, np.ndarray):\n",
|
|
" self._data = data.astype(dtype or data.dtype)\n",
|
|
" else:\n",
|
|
" raise TypeError(f\"Cannot create tensor from {type(data)}\")\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def data(self) -> np.ndarray:\n",
|
|
" \"\"\"Access underlying numpy array.\"\"\"\n",
|
|
" return self._data\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def shape(self) -> Tuple[int, ...]:\n",
|
|
" \"\"\"Get tensor shape.\"\"\"\n",
|
|
" return self._data.shape\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def size(self) -> int:\n",
|
|
" \"\"\"Get total number of elements.\"\"\"\n",
|
|
" return self._data.size\n",
|
|
" \n",
|
|
" @property\n",
|
|
" def dtype(self) -> np.dtype:\n",
|
|
" \"\"\"Get data type as numpy dtype.\"\"\"\n",
|
|
" return self._data.dtype\n",
|
|
" \n",
|
|
" def __repr__(self) -> str:\n",
|
|
" \"\"\"String representation.\"\"\"\n",
|
|
" return f\"Tensor({self._data.tolist()}, shape={self.shape}, dtype={self.dtype})\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"### \ud83e\uddea Test Your Tensor Class\n",
|
|
"\n",
|
|
"Once you implement the Tensor class above, run this cell to test it:\n",
|
|
"\"\"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Test the basic Tensor class\n",
|
|
"try:\n",
|
|
" print(\"=== Testing Tensor Creation ===\")\n",
|
|
" \n",
|
|
" # Scalar tensor\n",
|
|
" scalar = Tensor(5.0)\n",
|
|
" print(f\"Scalar: {scalar}\")\n",
|
|
" \n",
|
|
" # Vector tensor \n",
|
|
" vector = Tensor([1, 2, 3])\n",
|
|
" print(f\"Vector: {vector}\")\n",
|
|
" \n",
|
|
" # Matrix tensor\n",
|
|
" matrix = Tensor([[1, 2], [3, 4]])\n",
|
|
" print(f\"Matrix: {matrix}\")\n",
|
|
" \n",
|
|
" print(f\"\\nProperties:\")\n",
|
|
" print(f\"Matrix shape: {matrix.shape}\")\n",
|
|
" print(f\"Matrix size: {matrix.size}\")\n",
|
|
" print(f\"Matrix dtype: {matrix.dtype}\")\n",
|
|
" \n",
|
|
"except NotImplementedError as e:\n",
|
|
" print(f\"\u26a0\ufe0f {e}\")\n",
|
|
" print(\"Implement the Tensor class above first!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"## Step 2: Arithmetic Operations\n",
|
|
"\n",
|
|
"Now let's add the core arithmetic operations. These are essential for neural networks:\n",
|
|
"- **Addition**: `tensor + other` \n",
|
|
"- **Subtraction**: `tensor - other`\n",
|
|
"- **Multiplication**: `tensor * other`\n",
|
|
"- **Division**: `tensor / other`\n",
|
|
"\n",
|
|
"Each operation should handle both **tensor + tensor** and **tensor + scalar** cases.\n",
|
|
"\"\"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _add_arithmetic_methods():\n",
|
|
" \"\"\"\n",
|
|
" Add arithmetic operations to Tensor class.\n",
|
|
" \n",
|
|
" TODO: Implement arithmetic methods (__add__, __sub__, __mul__, __truediv__)\n",
|
|
" and their reverse operations (__radd__, __rsub__, etc.)\n",
|
|
" \"\"\"\n",
|
|
" \n",
|
|
" def __add__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Addition: tensor + other\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" def __sub__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Subtraction: tensor - other\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" def __mul__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Multiplication: tensor * other\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" def __truediv__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Division: tensor / other\"\"\"\n",
|
|
" raise NotImplementedError(\"Student implementation required\")\n",
|
|
" \n",
|
|
" # Add methods to Tensor class\n",
|
|
" Tensor.__add__ = __add__\n",
|
|
" Tensor.__sub__ = __sub__\n",
|
|
" Tensor.__mul__ = __mul__\n",
|
|
" Tensor.__truediv__ = __truediv__"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide \n",
|
|
"#| export\n",
|
|
"def _add_arithmetic_methods():\n",
|
|
" \"\"\"Add arithmetic operations to Tensor class.\"\"\"\n",
|
|
" \n",
|
|
" def __add__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Addition: tensor + other\"\"\"\n",
|
|
" if isinstance(other, Tensor):\n",
|
|
" return Tensor(self._data + other._data)\n",
|
|
" else: # scalar\n",
|
|
" return Tensor(self._data + other)\n",
|
|
" \n",
|
|
" def __sub__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Subtraction: tensor - other\"\"\"\n",
|
|
" if isinstance(other, Tensor):\n",
|
|
" return Tensor(self._data - other._data)\n",
|
|
" else: # scalar\n",
|
|
" return Tensor(self._data - other)\n",
|
|
" \n",
|
|
" def __mul__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Multiplication: tensor * other\"\"\"\n",
|
|
" if isinstance(other, Tensor):\n",
|
|
" return Tensor(self._data * other._data)\n",
|
|
" else: # scalar\n",
|
|
" return Tensor(self._data * other)\n",
|
|
" \n",
|
|
" def __truediv__(self, other: Union['Tensor', int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Division: tensor / other\"\"\"\n",
|
|
" if isinstance(other, Tensor):\n",
|
|
" return Tensor(self._data / other._data)\n",
|
|
" else: # scalar\n",
|
|
" return Tensor(self._data / other)\n",
|
|
" \n",
|
|
" def __radd__(self, other: Union[int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Reverse addition: scalar + tensor\"\"\"\n",
|
|
" return Tensor(other + self._data)\n",
|
|
" \n",
|
|
" def __rsub__(self, other: Union[int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Reverse subtraction: scalar - tensor\"\"\"\n",
|
|
" return Tensor(other - self._data)\n",
|
|
" \n",
|
|
" def __rmul__(self, other: Union[int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Reverse multiplication: scalar * tensor\"\"\"\n",
|
|
" return Tensor(other * self._data)\n",
|
|
" \n",
|
|
" def __rtruediv__(self, other: Union[int, float]) -> 'Tensor':\n",
|
|
" \"\"\"Reverse division: scalar / tensor\"\"\"\n",
|
|
" return Tensor(other / self._data)\n",
|
|
" \n",
|
|
" # Add methods to Tensor class\n",
|
|
" Tensor.__add__ = __add__\n",
|
|
" Tensor.__sub__ = __sub__\n",
|
|
" Tensor.__mul__ = __mul__\n",
|
|
" Tensor.__truediv__ = __truediv__\n",
|
|
" Tensor.__radd__ = __radd__\n",
|
|
" Tensor.__rsub__ = __rsub__\n",
|
|
" Tensor.__rmul__ = __rmul__\n",
|
|
" Tensor.__rtruediv__ = __rtruediv__\n",
|
|
"\n",
|
|
"# Call the function to add arithmetic methods\n",
|
|
"_add_arithmetic_methods()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"### \ud83e\uddea Test Your Arithmetic Operations\n",
|
|
"\n",
|
|
"Once you implement the arithmetic methods above, run this cell to test them:\n",
|
|
"\"\"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Test arithmetic operations\n",
|
|
"try:\n",
|
|
" print(\"=== Testing Arithmetic Operations ===\")\n",
|
|
" \n",
|
|
" a = Tensor([1, 2, 3])\n",
|
|
" b = Tensor([4, 5, 6])\n",
|
|
" \n",
|
|
" print(f\"a = {a}\")\n",
|
|
" print(f\"b = {b}\")\n",
|
|
" print()\n",
|
|
" \n",
|
|
" # Tensor + Tensor\n",
|
|
" print(f\"a + b = {a + b}\")\n",
|
|
" print(f\"a - b = {a - b}\")\n",
|
|
" print(f\"a * b = {a * b}\")\n",
|
|
" print(f\"a / b = {a / b}\")\n",
|
|
" print()\n",
|
|
" \n",
|
|
" # Tensor + Scalar\n",
|
|
" print(f\"a + 10 = {a + 10}\")\n",
|
|
" print(f\"a * 2 = {a * 2}\")\n",
|
|
" print()\n",
|
|
" \n",
|
|
" # Scalar + Tensor (reverse operations)\n",
|
|
" print(f\"10 + a = {10 + a}\")\n",
|
|
" print(f\"2 * a = {2 * a}\")\n",
|
|
" \n",
|
|
"except (NotImplementedError, AttributeError) as e:\n",
|
|
" print(f\"\u26a0\ufe0f {e}\")\n",
|
|
" print(\"Implement the arithmetic methods above first!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"## Step 3: Try the Export Process\n",
|
|
"\n",
|
|
"Now let's export our tensor code! In your terminal, run:\n",
|
|
"\n",
|
|
"```bash\n",
|
|
"python bin/tito.py sync --module tensor\n",
|
|
"```\n",
|
|
"\n",
|
|
"This will export the code marked with `#| export` to `tinytorch/core/tensor.py`.\n",
|
|
"\n",
|
|
"Then test it with:\n",
|
|
"\n",
|
|
"```bash\n",
|
|
"python bin/tito.py test --module tensor\n",
|
|
"```\n",
|
|
"\n",
|
|
"## Next Steps\n",
|
|
"\n",
|
|
"\ud83c\udf89 **Congratulations!** You've built the foundation of TinyTorch - the Tensor class. \n",
|
|
"\n",
|
|
"In the next modules, you'll add:\n",
|
|
"- **Automatic differentiation** (gradients)\n",
|
|
"- **Neural network layers**\n",
|
|
"- **Optimizers and training loops**\n",
|
|
"- **GPU acceleration**\n",
|
|
"\n",
|
|
"Each builds on this tensor foundation!\n",
|
|
"\"\"\""
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"name": "python",
|
|
"version": "3.8.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
} |