mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-07-21 20:44:04 -05:00
Vijay Janapa Reddi
621be58efa
- Add tensor_dev.ipynb converted from tensor_dev.py - Add activations_dev.ipynb converted from activations_dev.py These notebooks provide interactive learning environments for students to explore tensor operations and activation functions.
64 KiB
64 KiB
In [ ]:
#| default_exp core.tensor
#| export
import numpy as np
import sys
from typing import Union, List, Tuple, Optional, AnyIn [ ]:
print("🔥 TinyTorch Tensor Module")
print(f"NumPy version: {np.__version__}")
print(f"Python version: {sys.version_info.major}.{sys.version_info.minor}")
print("Ready to build tensors!")In [ ]:
#| export
class Tensor:
"""
TinyTorch Tensor: N-dimensional array with ML operations.
The fundamental data structure for all TinyTorch operations.
Wraps NumPy arrays with ML-specific functionality.
"""
def __init__(self, data: Union[int, float, List, np.ndarray], dtype: Optional[str] = None):
"""
Create a new tensor from data.
Args:
data: Input data (scalar, list, or numpy array)
dtype: Data type ('float32', 'int32', etc.). Defaults to auto-detect.
TODO: Implement tensor creation with proper type handling.
APPROACH:
1. Check the input data type using isinstance()
- If scalar (int/float): convert to numpy array
- If list: convert to numpy array
- If numpy array: use directly (but make a copy)
2. Handle the dtype parameter:
- If dtype is None: auto-detect the best type
- If dtype is specified: convert to that type
3. For auto-detection, prefer:
- int32 for integers (not int64)
- float32 for floats (not float64)
4. Store the final array in self._data
EXAMPLE:
>>> t1 = Tensor(5) # Creates int32 array
>>> t2 = Tensor(3.14) # Creates float32 array
>>> t3 = Tensor([1, 2, 3]) # Creates int32 array
>>> t4 = Tensor([1.0, 2.0], 'float64') # Forces float64
>>> t5 = Tensor(np.array([1, 2])) # Preserves numpy array
HINTS:
- Use isinstance(data, (int, float)) to check for scalars
- Use isinstance(data, list) for lists
- Use isinstance(data, np.ndarray) for numpy arrays
- For dtype conversion: np.array(data, dtype=dtype)
- Remember: we prefer float32 over float64 for ML efficiency
"""
### BEGIN SOLUTION
# Convert input to numpy array
if isinstance(data, (int, float, np.number)):
# Handle Python and NumPy scalars
if dtype is None:
# Auto-detect type: int for integers, float32 for floats
if isinstance(data, int) or (isinstance(data, np.number) and np.issubdtype(type(data), np.integer)):
dtype = 'int32'
else:
dtype = 'float32'
self._data = np.array(data, dtype=dtype)
elif isinstance(data, list):
# Let NumPy auto-detect type, then convert if needed
temp_array = np.array(data)
if dtype is None:
# Use NumPy's auto-detected type, but prefer float32 for floats
if temp_array.dtype == np.float64:
dtype = 'float32'
else:
dtype = str(temp_array.dtype)
self._data = np.array(data, dtype=dtype)
elif isinstance(data, np.ndarray):
# Already a numpy array
if dtype is None:
# Keep existing dtype, but prefer float32 for float64
if data.dtype == np.float64:
dtype = 'float32'
else:
dtype = str(data.dtype)
self._data = data.astype(dtype) if dtype != data.dtype else data.copy()
else:
# Try to convert unknown types
self._data = np.array(data, dtype=dtype)
### END SOLUTION
@property
def data(self) -> np.ndarray:
"""
Access underlying numpy array.
TODO: Return the stored numpy array.
HINT: Return self._data (the array you stored in __init__)
"""
### BEGIN SOLUTION
return self._data
### END SOLUTION
@property
def shape(self) -> Tuple[int, ...]:
"""
Get tensor shape.
TODO: Return the shape of the stored numpy array.
HINT: Use .shape attribute of the numpy array
EXAMPLE: Tensor([1, 2, 3]).shape should return (3,)
"""
### BEGIN SOLUTION
return self._data.shape
### END SOLUTION
@property
def size(self) -> int:
"""
Get total number of elements.
TODO: Return the total number of elements in the tensor.
HINT: Use .size attribute of the numpy array
EXAMPLE: Tensor([1, 2, 3]).size should return 3
"""
### BEGIN SOLUTION
return self._data.size
### END SOLUTION
@property
def dtype(self) -> np.dtype:
"""
Get data type as numpy dtype.
TODO: Return the data type of the stored numpy array.
HINT: Use .dtype attribute of the numpy array
EXAMPLE: Tensor([1, 2, 3]).dtype should return dtype('int32')
"""
### BEGIN SOLUTION
return self._data.dtype
### END SOLUTION
def __repr__(self) -> str:
"""
String representation.
TODO: Create a clear string representation of the tensor.
APPROACH:
1. Convert the numpy array to a list for readable output
2. Include the shape and dtype information
3. Format: "Tensor([data], shape=shape, dtype=dtype)"
EXAMPLE:
Tensor([1, 2, 3]) → "Tensor([1, 2, 3], shape=(3,), dtype=int32)"
HINTS:
- Use .tolist() to convert numpy array to list
- Include shape and dtype information
- Keep format consistent and readable
"""
### BEGIN SOLUTION
return f"Tensor({self._data.tolist()}, shape={self.shape}, dtype={self.dtype})"
### END SOLUTION
def add(self, other: 'Tensor') -> 'Tensor':
"""
Add two tensors element-wise.
TODO: Implement tensor addition.
APPROACH:
1. Add the numpy arrays using +
2. Return a new Tensor with the result
3. Handle broadcasting automatically
EXAMPLE:
Tensor([1, 2]) + Tensor([3, 4]) → Tensor([4, 6])
HINTS:
- Use self._data + other._data
- Return Tensor(result)
- NumPy handles broadcasting automatically
"""
### BEGIN SOLUTION
result = self._data + other._data
return Tensor(result)
### END SOLUTION
def multiply(self, other: 'Tensor') -> 'Tensor':
"""
Multiply two tensors element-wise.
TODO: Implement tensor multiplication.
APPROACH:
1. Multiply the numpy arrays using *
2. Return a new Tensor with the result
3. Handle broadcasting automatically
EXAMPLE:
Tensor([1, 2]) * Tensor([3, 4]) → Tensor([3, 8])
HINTS:
- Use self._data * other._data
- Return Tensor(result)
- This is element-wise, not matrix multiplication
"""
### BEGIN SOLUTION
result = self._data * other._data
return Tensor(result)
### END SOLUTION
def __add__(self, other: Union['Tensor', int, float]) -> 'Tensor':
"""
Addition operator overload for natural syntax: tensor + other.
TODO: Implement the + operator to enable natural addition syntax for tensors.
APPROACH:
1. Check if other is a Tensor or scalar (int/float)
2. If scalar, convert to Tensor for uniform handling
3. Delegate to the add() method for actual computation
4. Return the resulting Tensor
EXAMPLE:
>>> a = Tensor([1, 2, 3])
>>> b = Tensor([4, 5, 6])
>>> c = a + b # Element-wise addition
>>> print(c.data) # [5, 7, 9]
>>> d = a + 10 # Broadcasting with scalar
>>> print(d.data) # [11, 12, 13]
HINTS:
- Use isinstance(other, Tensor) to check type
- Convert scalars with Tensor(other)
- Delegate to self.add() for consistency
- Python automatically calls this for + operator
LEARNING CONNECTIONS:
- This is how torch.Tensor implements natural syntax
- Operator overloading makes math operations intuitive
- Broadcasting happens automatically in the add() method
"""
### BEGIN SOLUTION
if isinstance(other, Tensor):
return self.add(other)
else:
return self.add(Tensor(other))
### END SOLUTION
def __mul__(self, other: Union['Tensor', int, float]) -> 'Tensor':
"""
Multiplication operator overload for natural syntax: tensor * other.
TODO: Implement the * operator for element-wise multiplication (Hadamard product).
APPROACH:
1. Check if other is a Tensor or scalar (int/float)
2. If scalar, convert to Tensor for uniform handling
3. Delegate to the multiply() method for computation
4. Return the resulting Tensor
EXAMPLE:
>>> a = Tensor([2, 3, 4])
>>> b = Tensor([5, 6, 7])
>>> c = a * b # Element-wise multiplication
>>> print(c.data) # [10, 18, 28]
>>> d = a * 3 # Scalar multiplication
>>> print(d.data) # [6, 9, 12]
HINTS:
- This is element-wise, NOT matrix multiplication
- For matrix multiplication, use @ operator (matmul)
- Scalar multiplication broadcasts automatically
- Delegate to self.multiply() for consistency
LEARNING CONNECTIONS:
- In PyTorch: * is element-wise, @ is matrix multiplication
- Neural networks use this for element-wise activations
- Scalar multiplication is used for learning rate scaling
"""
### BEGIN SOLUTION
if isinstance(other, Tensor):
return self.multiply(other)
else:
return self.multiply(Tensor(other))
### END SOLUTION
def __sub__(self, other: Union['Tensor', int, float]) -> 'Tensor':
"""
Subtraction operator overload for natural syntax: tensor - other.
TODO: Implement the - operator for element-wise subtraction.
APPROACH:
1. Check if other is a Tensor or scalar
2. Access the underlying NumPy arrays with ._data
3. Perform subtraction using NumPy's broadcasting
4. Wrap the result in a new Tensor and return
EXAMPLE:
>>> a = Tensor([10, 20, 30])
>>> b = Tensor([3, 5, 7])
>>> c = a - b # Element-wise subtraction
>>> print(c.data) # [7, 15, 23]
>>> d = a - 5 # Broadcasting with scalar
>>> print(d.data) # [5, 15, 25]
HINTS:
- Access data with self._data and other._data
- NumPy handles broadcasting automatically
- Always return a new Tensor (immutability)
- Handle both Tensor and scalar cases
LEARNING CONNECTIONS:
- Used in gradient computation (loss - target)
- Batch normalization uses mean subtraction
- Residual connections compute (output - input)
"""
### BEGIN SOLUTION
if isinstance(other, Tensor):
result = self._data - other._data
else:
result = self._data - other
return Tensor(result)
### END SOLUTION
def __truediv__(self, other: Union['Tensor', int, float]) -> 'Tensor':
"""
Division operator overload for natural syntax: tensor / other.
TODO: Implement the / operator for element-wise division.
APPROACH:
1. Check if other is a Tensor or scalar
2. Access the underlying NumPy arrays
3. Perform division (watch for division by zero!)
4. Wrap the result in a new Tensor and return
EXAMPLE:
>>> a = Tensor([10, 20, 30])
>>> b = Tensor([2, 4, 5])
>>> c = a / b # Element-wise division
>>> print(c.data) # [5.0, 5.0, 6.0]
>>> d = a / 10 # Scalar division
>>> print(d.data) # [1.0, 2.0, 3.0]
HINTS:
- NumPy handles division by zero → inf/nan
- Results are always float type
- Consider using safe_divide() for production
- Broadcasting rules apply here too
LEARNING CONNECTIONS:
- Normalization often uses division (x / std)
- Softmax involves division by sum
- Learning rate scheduling uses division
- Numerical stability is critical in ML
"""
### BEGIN SOLUTION
if isinstance(other, Tensor):
result = self._data / other._data
else:
result = self._data / other
return Tensor(result)
### END SOLUTION
def mean(self) -> 'Tensor':
"""Computes the mean of the tensor's elements."""
return Tensor(np.mean(self.data))
# --- Matmul ---
def matmul(self, other: 'Tensor') -> 'Tensor':
"""
Perform matrix multiplication between two tensors.
TODO: Implement matrix multiplication.
APPROACH:
1. Use np.matmul() to perform matrix multiplication
2. Return a new Tensor with the result
3. Handle broadcasting automatically
EXAMPLE:
Tensor([[1, 2], [3, 4]]) @ Tensor([[5, 6], [7, 8]]) → Tensor([[19, 22], [43, 50]])
HINTS:
- Use np.matmul(self._data, other._data)
- Return Tensor(result)
- This is matrix multiplication, not element-wise multiplication
"""
### BEGIN SOLUTION
result = np.matmul(self._data, other._data)
return Tensor(result)
### END SOLUTION
# --- Error Handling ---
def validate_operation(self, other: 'Tensor', operation: str) -> None:
"""
Validate tensor compatibility for operations with informative error messages.
TODO: Implement validation for tensor operations that checks shape compatibility
and provides educational error messages when operations would fail.
APPROACH:
1. Check if the operation is element-wise or matrix multiplication
2. For element-wise ops, verify broadcasting compatibility
3. For matmul, verify matrix dimension alignment
4. Raise informative ValueError with hints if incompatible
EXAMPLE:
>>> a = Tensor([[1, 2, 3]]) # Shape: (1, 3)
>>> b = Tensor([[1], [2]]) # Shape: (2, 1)
>>> a.validate_operation(b, 'add') # Should pass (broadcasting)
>>> a.validate_operation(b, 'matmul') # Should raise with helpful message
HINTS:
- Use np.broadcast_shapes() to check broadcasting compatibility
- For matmul, check that self.shape[-1] == other.shape[-2]
- Include shapes in error messages to help debugging
- Suggest fixes in the error message
LEARNING CONNECTIONS:
- PyTorch raises similar errors with shape information
- Good error messages accelerate learning and debugging
- Understanding shape compatibility is crucial for ML
"""
### BEGIN SOLUTION
if operation in ['add', 'mul', 'sub', 'div']:
# Check broadcasting compatibility for element-wise operations
try:
np.broadcast_shapes(self.shape, other.shape)
except ValueError:
raise ValueError(
f"Cannot {operation} tensors with incompatible shapes: "
f"{self.shape} and {other.shape}. "
f"Hint: Shapes must be broadcastable. Either dimensions match "
f"or one of them is 1. Consider reshaping with tensor.reshape()."
)
elif operation == 'matmul':
# Check matrix multiplication compatibility
if len(self.shape) == 0 or len(other.shape) == 0:
raise ValueError(
f"Cannot perform matrix multiplication with scalar tensors. "
f"Shapes: {self.shape} @ {other.shape}. "
f"Hint: Use regular multiplication (*) for scalar operations."
)
if self.shape[-1] != other.shape[-2 if len(other.shape) > 1 else 0]:
raise ValueError(
f"Cannot perform matrix multiplication with incompatible shapes: "
f"{self.shape} @ {other.shape}. "
f"Hint: For A @ B, A's last dimension ({self.shape[-1]}) must match "
f"B's second-to-last dimension ({other.shape[-2 if len(other.shape) > 1 else 0]}). "
f"Consider transposing with tensor.T or reshaping."
)
### END SOLUTION
def safe_divide(self, other: Union['Tensor', int, float]) -> 'Tensor':
"""
Perform division with zero-division protection and informative warnings.
TODO: Implement safe division that handles division by zero gracefully
and provides educational feedback about numerical stability.
APPROACH:
1. Check for zero values in the divisor
2. Use np.where to handle zero divisions appropriately
3. Optionally warn about near-zero values that might cause instability
4. Return result with inf/nan handled according to NumPy conventions
EXAMPLE:
>>> a = Tensor([1, 2, 3, 4])
>>> b = Tensor([2, 0, 1, 0])
>>> result = a.safe_divide(b) # Handles zeros gracefully
>>> print(result.data) # [0.5, inf, 3.0, inf] with warning
HINTS:
- Use np.where(divisor == 0, np.inf, normal_division)
- Consider np.errstate(divide='ignore') context manager
- Check for small values with np.abs(divisor) < epsilon
- Return informative warning messages for learning
LEARNING CONNECTIONS:
- Numerical stability is critical in ML (gradient explosions)
- Many ML bugs come from division by zero in normalizations
- Production systems need robust error handling
"""
### BEGIN SOLUTION
other_data = other._data if isinstance(other, Tensor) else other
# Check for exact zeros
if np.any(other_data == 0):
import warnings
warnings.warn(
"Division by zero detected! Results will contain inf values. "
"In ML, this often happens in normalization or loss calculations. "
"Consider adding a small epsilon (e.g., 1e-8) to prevent instability.",
category=RuntimeWarning
)
# Check for near-zero values that might cause instability
epsilon = 1e-10
if np.any(np.abs(other_data) < epsilon):
import warnings
warnings.warn(
f"Near-zero divisor detected (values < {epsilon}). "
"This may cause numerical instability in gradients. "
"Consider using stabilized versions of your operations.",
category=RuntimeWarning
)
# Perform division with NumPy's default handling (inf for division by zero)
with np.errstate(divide='ignore', invalid='ignore'):
result = self._data / other_data
return Tensor(result)
### END SOLUTIONIn [ ]:
# Test tensor creation immediately after implementation
print("🔬 Unit Test: Tensor Creation...")
# Test basic tensor creation
try:
# Test scalar
scalar = Tensor(5.0)
assert hasattr(scalar, '_data'), "Tensor should have _data attribute"
assert scalar._data.shape == (), f"Scalar should have shape (), got {scalar._data.shape}"
print("✅ Scalar creation works")
# Test vector
vector = Tensor([1, 2, 3])
assert vector._data.shape == (3,), f"Vector should have shape (3,), got {vector._data.shape}"
print("✅ Vector creation works")
# Test matrix
matrix = Tensor([[1, 2], [3, 4]])
assert matrix._data.shape == (2, 2), f"Matrix should have shape (2, 2), got {matrix._data.shape}"
print("✅ Matrix creation works")
print("📈 Progress: Tensor Creation ✓")
except Exception as e:
print(f"❌ Tensor creation test failed: {e}")
raise
print("🎯 Tensor creation behavior:")
print(" Converts data to NumPy arrays")
print(" Preserves shape and data type")
print(" Stores in _data attribute")In [ ]:
# Test tensor properties immediately after implementation
print("🔬 Unit Test: Tensor Properties...")
# Test properties with simple examples
try:
# Test with a simple matrix
tensor = Tensor([[1, 2, 3], [4, 5, 6]])
# Test shape property
assert tensor.shape == (2, 3), f"Shape should be (2, 3), got {tensor.shape}"
print("✅ Shape property works")
# Test size property
assert tensor.size == 6, f"Size should be 6, got {tensor.size}"
print("✅ Size property works")
# Test data property
assert np.array_equal(tensor.data, np.array([[1, 2, 3], [4, 5, 6]])), "Data property should return numpy array"
print("✅ Data property works")
# Test dtype property
assert tensor.dtype in [np.int32, np.int64], f"Dtype should be int32 or int64, got {tensor.dtype}"
print("✅ Dtype property works")
print("📈 Progress: Tensor Properties ✓")
except Exception as e:
print(f"❌ Tensor properties test failed: {e}")
raise
print("🎯 Tensor properties behavior:")
print(" shape: Returns tuple of dimensions")
print(" size: Returns total number of elements")
print(" data: Returns underlying NumPy array")
print(" dtype: Returns NumPy data type")In [ ]:
# Test tensor arithmetic immediately after implementation
print("🔬 Unit Test: Tensor Arithmetic...")
# Test basic arithmetic with simple examples
try:
# Test addition
a = Tensor([1, 2, 3])
b = Tensor([4, 5, 6])
result = a + b
expected = np.array([5, 7, 9])
assert np.array_equal(result.data, expected), f"Addition failed: expected {expected}, got {result.data}"
print("✅ Addition works")
# Test scalar addition
result_scalar = a + 10
expected_scalar = np.array([11, 12, 13])
assert np.array_equal(result_scalar.data, expected_scalar), f"Scalar addition failed: expected {expected_scalar}, got {result_scalar.data}"
print("✅ Scalar addition works")
# Test multiplication
result_mul = a * b
expected_mul = np.array([4, 10, 18])
assert np.array_equal(result_mul.data, expected_mul), f"Multiplication failed: expected {expected_mul}, got {result_mul.data}"
print("✅ Multiplication works")
# Test scalar multiplication
result_scalar_mul = a * 2
expected_scalar_mul = np.array([2, 4, 6])
assert np.array_equal(result_scalar_mul.data, expected_scalar_mul), f"Scalar multiplication failed: expected {expected_scalar_mul}, got {result_scalar_mul.data}"
print("✅ Scalar multiplication works")
print("📈 Progress: Tensor Arithmetic ✓")
except Exception as e:
print(f"❌ Tensor arithmetic test failed: {e}")
raise
print("🎯 Tensor arithmetic behavior:")
print(" Element-wise operations on tensors")
print(" Broadcasting with scalars")
print(" Returns new Tensor objects")In [ ]:
def test_unit_tensor_creation():
"""Comprehensive test of tensor creation with all data types and shapes."""
print("🔬 Testing comprehensive tensor creation...")
# Test scalar creation
scalar_int = Tensor(42)
assert scalar_int.shape == ()
# Test vector creation
vector_int = Tensor([1, 2, 3])
assert vector_int.shape == (3,)
# Test matrix creation
matrix_2x2 = Tensor([[1, 2], [3, 4]])
assert matrix_2x2.shape == (2, 2)
print("✅ Tensor creation tests passed!")
# Run the test
test_unit_tensor_creation()In [ ]:
def test_unit_tensor_properties():
"""Comprehensive test of tensor properties (shape, size, dtype, data access)."""
print("🔬 Testing comprehensive tensor properties...")
tensor = Tensor([[1, 2, 3], [4, 5, 6]])
# Test shape property
assert tensor.shape == (2, 3)
# Test size property
assert tensor.size == 6
# Test data property
assert np.array_equal(tensor.data, np.array([[1, 2, 3], [4, 5, 6]]))
# Test dtype property
assert tensor.dtype in [np.int32, np.int64]
print("✅ Tensor properties tests passed!")
# Run the test
test_unit_tensor_properties()In [ ]:
def test_unit_tensor_arithmetic():
"""Comprehensive test of tensor arithmetic operations."""
print("🔬 Testing comprehensive tensor arithmetic...")
a = Tensor([1, 2, 3])
b = Tensor([4, 5, 6])
# Test addition
c = a + b
expected = np.array([5, 7, 9])
assert np.array_equal(c.data, expected)
# Test multiplication
d = a * b
expected = np.array([4, 10, 18])
assert np.array_equal(d.data, expected)
# Test subtraction
e = b - a
expected = np.array([3, 3, 3])
assert np.array_equal(e.data, expected)
# Test division
f = b / a
expected = np.array([4.0, 2.5, 2.0])
assert np.allclose(f.data, expected)
print("✅ Tensor arithmetic tests passed!")
# Run the test
test_unit_tensor_arithmetic()In [ ]:
def test_module_tensor_numpy_integration():
"""
Integration test for tensor operations with NumPy arrays.
Tests that tensors properly integrate with NumPy operations and maintain
compatibility with the scientific Python ecosystem.
"""
print("🔬 Running Integration Test: Tensor-NumPy Integration...")
# Test 1: Tensor from NumPy array
numpy_array = np.array([[1, 2, 3], [4, 5, 6]])
tensor_from_numpy = Tensor(numpy_array)
assert tensor_from_numpy.shape == (2, 3), "Tensor should preserve NumPy array shape"
assert np.array_equal(tensor_from_numpy.data, numpy_array), "Tensor should preserve NumPy array data"
# Test 2: Tensor arithmetic with NumPy-compatible operations
a = Tensor([1.0, 2.0, 3.0])
b = Tensor([4.0, 5.0, 6.0])
# Test operations that would be used in neural networks
dot_product_result = np.dot(a.data, b.data) # Common in layers
assert np.isclose(dot_product_result, 32.0), "Dot product should work with tensor data"
# Test 3: Broadcasting compatibility
matrix = Tensor([[1, 2], [3, 4]])
scalar = Tensor(10)
result = matrix + scalar
expected = np.array([[11, 12], [13, 14]])
assert np.array_equal(result.data, expected), "Broadcasting should work like NumPy"
# Test 4: Integration with scientific computing patterns
data = Tensor([1, 4, 9, 16, 25])
sqrt_result = Tensor(np.sqrt(data.data)) # Using NumPy functions on tensor data
expected_sqrt = np.array([1., 2., 3., 4., 5.])
assert np.allclose(sqrt_result.data, expected_sqrt), "Should integrate with NumPy functions"
print("✅ Integration Test Passed: Tensor-NumPy integration works correctly.")
# Run the integration test
test_module_tensor_numpy_integration()