github-starred/TinyTorch
2
0
Fork 0
mirror of https://github.com/MLSysBook/TinyTorch.git synced 2026-03-13 17:33:32 -05:00
Code Issues 7 Packages Projects Releases Wiki Activity

Fix syntax errors in layers, networks, and cnn modules

Browse Source 
- Fixed indentation issues in 03_layers/layers_dev.py
- Fixed indentation issues in 04_networks/networks_dev.py
- Fixed indentation issues in 05_cnn/cnn_dev.py
- Removed orphaned except/raise statements
- 06_dataloader still has some complex indentation issues to resolve
This commit is contained in:
Vijay Janapa Reddi
2025-07-13 18:13:36 -04:00
parent 4ad611383a
commit 5bcda83bef
5 changed files with 155 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
modules/source/02_activations/activations_dev.py
View File

@@ -231,10 +231,10 @@ def test_relu_activation():
"""Test ReLU activation function"""
print("🔬 Unit Test: ReLU Activation...")
# Create ReLU instance
relu = ReLU()
# Create ReLU instance
relu = ReLU()
# Test with mixed positive/negative values
# Test with mixed positive/negative values
test_input = Tensor([[-2, -1, 0, 1, 2]])
result = relu(test_input)
expected = np.array([[0, 0, 0, 1, 2]])
@@ -370,7 +370,7 @@ def test_sigmoid_activation():
print("🔬 Unit Test: Sigmoid Activation...")
# Create Sigmoid instance
sigmoid = Sigmoid()
sigmoid = Sigmoid()
# Test with known values
test_input = Tensor([[0]])
@@ -516,7 +516,7 @@ def test_tanh_activation():
print("🔬 Unit Test: Tanh Activation...")
# Create Tanh instance
tanh = Tanh()
tanh = Tanh()
# Test with zero (should be 0)
test_input = Tensor([[0]])
@@ -678,7 +678,7 @@ def test_softmax_activation():
print("🔬 Unit Test: Softmax Activation...")
# Create Softmax instance
softmax = Softmax()
softmax = Softmax()
# Test with simple input
test_input = Tensor([[1, 2, 3]])
@@ -751,9 +751,9 @@ def test_activations_integration():
print("🔬 Unit Test: Activation Functions Integration...")
# Create instances of all activation functions
relu = ReLU()
sigmoid = Sigmoid()
tanh = Tanh()
relu = ReLU()
sigmoid = Sigmoid()
tanh = Tanh()
softmax = Softmax()
# Test data: simulating neural network layer outputs
@@ -791,7 +791,7 @@ def test_activations_integration():
# Test Softmax properties
softmax_sum = np.sum(softmax_result.data)
assert abs(softmax_sum - 1.0) < 1e-6, "Softmax outputs should sum to 1"
# Test chaining activations (realistic neural network scenario)
# Hidden layer with ReLU
hidden_output = relu(test_data)
@@ -809,14 +809,14 @@ def test_activations_integration():
# Test with batch data (multiple samples)
batch_data = Tensor([
[-2, -1, 0, 1, 2],
[1, 2, 3, 4, 5],
[-1, 0, 1, 2, 3]
[-2, -1, 0, 1, 2],
[1, 2, 3, 4, 5],
[-1, 0, 1, 2, 3]
])
batch_softmax = softmax(batch_data)
# Each row should sum to 1
# Each row should sum to 1
for i in range(batch_data.shape[0]):
row_sum = np.sum(batch_softmax.data[i])
assert abs(row_sum - 1.0) < 1e-6, f"Batch row {i} should sum to 1"
@@ -829,58 +829,58 @@ def test_activations_integration():
print(f"✅ Ready for neural network integration!")
# Run the integration test
test_activations_integration()
test_activations_integration()
# %% [markdown]
"""
## 🎯 Module Summary: Activation Functions Mastery!
Congratulations! You've successfully implemented all four essential activation functions:
Congratulations! You've successfully implemented all four essential activation functions:
### ✅ What You've Built
- **ReLU**: The foundation of modern deep learning with sparsity and efficiency
- **Sigmoid**: Classic activation for binary classification and probability outputs
- **Tanh**: Zero-centered activation with better gradient properties
- **Softmax**: Probability distribution for multi-class classification
- **ReLU**: The foundation of modern deep learning with sparsity and efficiency
- **Sigmoid**: Classic activation for binary classification and probability outputs
- **Tanh**: Zero-centered activation with better gradient properties
- **Softmax**: Probability distribution for multi-class classification
### ✅ Key Learning Outcomes
- **Understanding**: Why nonlinearity is essential for neural networks
- **Implementation**: Built activation functions from scratch using NumPy
- **Testing**: Progressive validation with immediate feedback after each function
- **Integration**: Saw how activations work together in neural networks
- **Real-world context**: Understanding where each activation is used
- **Understanding**: Why nonlinearity is essential for neural networks
- **Implementation**: Built activation functions from scratch using NumPy
- **Testing**: Progressive validation with immediate feedback after each function
- **Integration**: Saw how activations work together in neural networks
- **Real-world context**: Understanding where each activation is used
### ✅ Mathematical Mastery
- **ReLU**: f(x) = max(0, x) - Simple but powerful
- **Sigmoid**: f(x) = 1/(1 + e^(-x)) - Maps to (0,1)
- **Tanh**: f(x) = tanh(x) - Zero-centered, maps to (-1,1)
- **Softmax**: f(x_i) = e^(x_i)/Σ(e^(x_j)) - Probability distribution
- **ReLU**: f(x) = max(0, x) - Simple but powerful
- **Sigmoid**: f(x) = 1/(1 + e^(-x)) - Maps to (0,1)
- **Tanh**: f(x) = tanh(x) - Zero-centered, maps to (-1,1)
- **Softmax**: f(x_i) = e^(x_i)/Σ(e^(x_j)) - Probability distribution
### ✅ Professional Skills Developed
- **Numerical stability**: Handling overflow and underflow
- **API design**: Consistent interfaces across all functions
- **Testing discipline**: Immediate validation after each implementation
- **Integration thinking**: Understanding how components work together
- **Numerical stability**: Handling overflow and underflow
- **API design**: Consistent interfaces across all functions
- **Testing discipline**: Immediate validation after each implementation
- **Integration thinking**: Understanding how components work together
### ✅ Ready for Next Steps
Your activation functions are now ready to power:
- **Dense layers**: Linear transformations with nonlinear activations
- **Convolutional layers**: Spatial feature extraction with ReLU
- **Network architectures**: Complete neural networks with proper activations
- **Training**: Gradient computation through activation functions
Your activation functions are now ready to power:
- **Dense layers**: Linear transformations with nonlinear activations
- **Convolutional layers**: Spatial feature extraction with ReLU
- **Network architectures**: Complete neural networks with proper activations
- **Training**: Gradient computation through activation functions
### 🔗 Connection to Real ML Systems
Your implementations mirror production systems:
- **PyTorch**: `torch.nn.ReLU()`, `torch.nn.Sigmoid()`, `torch.nn.Tanh()`, `torch.nn.Softmax()`
- **TensorFlow**: `tf.nn.relu()`, `tf.nn.sigmoid()`, `tf.nn.tanh()`, `tf.nn.softmax()`
- **Industry applications**: Every major deep learning model uses these functions
Your implementations mirror production systems:
- **PyTorch**: `torch.nn.ReLU()`, `torch.nn.Sigmoid()`, `torch.nn.Tanh()`, `torch.nn.Softmax()`
- **TensorFlow**: `tf.nn.relu()`, `tf.nn.sigmoid()`, `tf.nn.tanh()`, `tf.nn.softmax()`
- **Industry applications**: Every major deep learning model uses these functions
### 🎯 The Power of Nonlinearity
You've unlocked the key to deep learning:
- **Before**: Linear models limited to simple patterns
- **After**: Nonlinear models can learn any pattern (universal approximation)
You've unlocked the key to deep learning:
- **Before**: Linear models limited to simple patterns
- **After**: Nonlinear models can learn any pattern (universal approximation)
**Next Module**: Layers - Building blocks that combine your tensors and activations into powerful transformations!
**Next Module**: Layers - Building blocks that combine your tensors and activations into powerful transformations!
Your activation functions are the key to neural network intelligence. Now let's build the layers that use them!
Your activation functions are the key to neural network intelligence. Now let's build the layers that use them!
"""

8
modules/source/03_layers/layers_dev.py
View File

@@ -46,8 +46,8 @@ except ImportError:
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor'))
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '02_activations'))
try:
from tensor_dev import Tensor
from activations_dev import ReLU, Sigmoid, Tanh, Softmax
from tensor_dev import Tensor
from activations_dev import ReLU, Sigmoid, Tanh, Softmax
except ImportError:
# If the local modules are not available, use relative imports
from ..tensor.tensor_dev import Tensor
@@ -538,7 +538,7 @@ def test_dense_layer():
scaled_output = layer(scaled_input)
# Due to bias, this won't be exactly 2*output, but the linear part should scale
print("✅ Dense layer tests passed!")
print("✅ Dense layer tests passed!")
print(f"✅ Correct weight and bias initialization")
print(f"✅ Forward pass produces correct shapes")
print(f"✅ Batch processing works correctly")
@@ -582,7 +582,7 @@ def test_layer_activation_integration():
# Create layer and activation functions
layer = Dense(input_size=4, output_size=3)
relu = ReLU()
relu = ReLU()
sigmoid = Sigmoid()
tanh = Tanh()
softmax = Softmax()

63
modules/source/04_networks/networks_dev.py
View File

@@ -560,7 +560,7 @@ try:
classifier = create_mlp(input_size=3, hidden_sizes=[4], output_size=3, output_activation=Softmax)
# Test with sample data
x = Tensor([[1.0, 2.0, 3.0]])
x = Tensor([[1.0, 2.0, 3.0]])
# Test ReLU network
y_relu = relu_net(x)
@@ -575,9 +575,9 @@ try:
# Test multi-class classifier
y_multi = classifier(x)
assert y_multi.shape == (1, 3), "Multi-class classifier should work"
# Check softmax properties
assert abs(np.sum(y_multi.data) - 1.0) < 1e-6, "Softmax outputs should sum to 1"
# Check softmax properties
assert abs(np.sum(y_multi.data) - 1.0) < 1e-6, "Softmax outputs should sum to 1"
print("✅ Multi-class classifier with Softmax works correctly")
# Test different architectures
@@ -595,7 +595,7 @@ try:
print("✅ All network architectures work correctly")
except Exception as e:
except Exception as e:
print(f"❌ Architecture test failed: {e}")
raise
@@ -643,18 +643,18 @@ try:
iris_classifier = create_mlp(input_size=4, hidden_sizes=[8, 6], output_size=3, output_activation=Softmax)
# Simulate iris features: [sepal_length, sepal_width, petal_length, petal_width]
iris_samples = Tensor([
iris_samples = Tensor([
[5.1, 3.5, 1.4, 0.2], # Setosa
[7.0, 3.2, 4.7, 1.4], # Versicolor
[6.3, 3.3, 6.0, 2.5] # Virginica
])
iris_predictions = iris_classifier(iris_samples)
iris_predictions = iris_classifier(iris_samples)
assert iris_predictions.shape == (3, 3), "Iris classifier should output 3 classes for 3 samples"
# Check softmax properties
row_sums = np.sum(iris_predictions.data, axis=1)
assert np.allclose(row_sums, 1.0), "Each prediction should sum to 1"
row_sums = np.sum(iris_predictions.data, axis=1)
assert np.allclose(row_sums, 1.0), "Each prediction should sum to 1"
print("✅ Multi-class classification works correctly")
# Test 2: Regression Task (Housing prices)
@@ -691,39 +691,38 @@ try:
# Test 4: Network Composition
print("\n4. Network Composition Test:")
# Create a feature extractor and classifier separately
feature_extractor = Sequential([
Dense(input_size=10, output_size=5),
ReLU(),
Dense(input_size=5, output_size=3),
ReLU()
])
classifier_head = Sequential([
Dense(input_size=3, output_size=2),
Softmax()
])
feature_extractor = Sequential([
Dense(input_size=10, output_size=5),
ReLU(),
Dense(input_size=5, output_size=3),
ReLU()
])
classifier_head = Sequential([
Dense(input_size=3, output_size=2),
Softmax()
])
# Test composition
raw_data = Tensor(np.random.randn(5, 10))
features = feature_extractor(raw_data)
final_predictions = classifier_head(features)
features = feature_extractor(raw_data)
final_predictions = classifier_head(features)
assert features.shape == (5, 3), "Feature extractor should output 3 features"
assert final_predictions.shape == (5, 2), "Classifier should output 2 classes"
row_sums = np.sum(final_predictions.data, axis=1)
row_sums = np.sum(final_predictions.data, axis=1)
assert np.allclose(row_sums, 1.0), "Composed network predictions should be valid"
print("✅ Network composition works correctly")
print("\n🎉 Integration test passed! Your networks work correctly for:")
print(" • Multi-class classification (Iris flowers)")
print(" • Regression tasks (housing prices)")
print(" • Multi-class classification (Iris flowers)")
print(" • Regression tasks (housing prices)")
print(" • Deep learning architectures")
print(" • Network composition and feature extraction")
except Exception as e:
print(f"❌ Integration test failed: {e}")
raise
except Exception as e:
print(f"❌ Integration test failed: {e}")
print("📈 Final Progress: Complete network architectures ready for real ML applications!")

94
modules/source/05_cnn/cnn_dev.py
View File

@@ -607,50 +607,50 @@ try:
print("\n1. Simple CNN Pipeline Test:")
# Create pipeline: Conv2D → ReLU → Flatten → Dense
conv = Conv2D(kernel_size=(2, 2))
relu = ReLU()
dense = Dense(input_size=4, output_size=3)
# Input image
image = Tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
# Forward pass
conv = Conv2D(kernel_size=(2, 2))
relu = ReLU()
dense = Dense(input_size=4, output_size=3)
# Input image
image = Tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
# Forward pass
features = conv(image) # (3,3) → (2,2)
activated = relu(features) # (2,2) → (2,2)
flattened = flatten(activated) # (2,2) → (1,4)
output = dense(flattened) # (1,4) → (1,3)
assert features.shape == (2, 2), f"Conv output shape wrong: {features.shape}"
assert activated.shape == (2, 2), f"ReLU output shape wrong: {activated.shape}"
assert flattened.shape == (1, 4), f"Flatten output shape wrong: {flattened.shape}"
assert output.shape == (1, 3), f"Dense output shape wrong: {output.shape}"
assert features.shape == (2, 2), f"Conv output shape wrong: {features.shape}"
assert activated.shape == (2, 2), f"ReLU output shape wrong: {activated.shape}"
assert flattened.shape == (1, 4), f"Flatten output shape wrong: {flattened.shape}"
assert output.shape == (1, 3), f"Dense output shape wrong: {output.shape}"
print("✅ Simple CNN pipeline works correctly")
# Test 2: Multi-layer CNN
print("\n2. Multi-layer CNN Test:")
# Create deeper pipeline: Conv2D → ReLU → Conv2D → ReLU → Flatten → Dense
conv1 = Conv2D(kernel_size=(2, 2))
relu1 = ReLU()
conv2 = Conv2D(kernel_size=(2, 2))
relu2 = ReLU()
conv1 = Conv2D(kernel_size=(2, 2))
relu1 = ReLU()
conv2 = Conv2D(kernel_size=(2, 2))
relu2 = ReLU()
dense_multi = Dense(input_size=9, output_size=2)
# Larger input for multi-layer processing
# Larger input for multi-layer processing
large_image = Tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]])
# Forward pass
# Forward pass
h1 = conv1(large_image) # (5,5) → (4,4)
h2 = relu1(h1) # (4,4) → (4,4)
h3 = conv2(h2) # (4,4) → (3,3)
h4 = relu2(h3) # (3,3) → (3,3)
h5 = flatten(h4) # (3,3) → (1,9)
output_multi = dense_multi(h5) # (1,9) → (1,2)
assert h1.shape == (4, 4), f"Conv1 output wrong: {h1.shape}"
assert h3.shape == (3, 3), f"Conv2 output wrong: {h3.shape}"
assert h5.shape == (1, 9), f"Flatten output wrong: {h5.shape}"
assert h1.shape == (4, 4), f"Conv1 output wrong: {h1.shape}"
assert h3.shape == (3, 3), f"Conv2 output wrong: {h3.shape}"
assert h5.shape == (1, 9), f"Flatten output wrong: {h5.shape}"
assert output_multi.shape == (1, 2), f"Final output wrong: {output_multi.shape}"
print("✅ Multi-layer CNN works correctly")
@@ -660,29 +660,29 @@ try:
# Simulate digit classification with 8x8 image
digit_image = Tensor([[1, 0, 0, 1, 1, 0, 0, 1],
[0, 1, 0, 1, 1, 0, 1, 0],
[0, 0, 1, 1, 1, 1, 0, 0],
[1, 1, 1, 0, 0, 1, 1, 1],
[1, 0, 0, 1, 1, 0, 0, 1],
[0, 1, 1, 0, 0, 1, 1, 0],
[0, 0, 1, 1, 1, 1, 0, 0],
[1, 1, 0, 0, 0, 0, 1, 1]])
# CNN for digit classification
[0, 1, 0, 1, 1, 0, 1, 0],
[0, 0, 1, 1, 1, 1, 0, 0],
[1, 1, 1, 0, 0, 1, 1, 1],
[1, 0, 0, 1, 1, 0, 0, 1],
[0, 1, 1, 0, 0, 1, 1, 0],
[0, 0, 1, 1, 1, 1, 0, 0],
[1, 1, 0, 0, 0, 0, 1, 1]])
# CNN for digit classification
feature_extractor = Conv2D(kernel_size=(3, 3)) # (8,8) → (6,6)
activation = ReLU()
classifier = Dense(input_size=36, output_size=10) # 10 digit classes
# Forward pass
features = feature_extractor(digit_image)
activated_features = activation(features)
activation = ReLU()
classifier = Dense(input_size=36, output_size=10) # 10 digit classes
# Forward pass
features = feature_extractor(digit_image)
activated_features = activation(features)
feature_vector = flatten(activated_features)
digit_scores = classifier(feature_vector)
assert features.shape == (6, 6), f"Feature extraction shape wrong: {features.shape}"
assert feature_vector.shape == (1, 36), f"Feature vector shape wrong: {feature_vector.shape}"
assert digit_scores.shape == (1, 10), f"Digit scores shape wrong: {digit_scores.shape}"
digit_scores = classifier(feature_vector)
assert features.shape == (6, 6), f"Feature extraction shape wrong: {features.shape}"
assert feature_vector.shape == (1, 36), f"Feature vector shape wrong: {feature_vector.shape}"
assert digit_scores.shape == (1, 10), f"Digit scores shape wrong: {digit_scores.shape}"
print("✅ Image classification scenario works correctly")
# Test 4: Feature Extraction and Composition

51
modules/source/06_dataloader/dataloader_dev.py
View File

@@ -760,23 +760,23 @@ try:
print("✅ SimpleDataset basic properties work correctly")
# Test sample access
data, label = dataset[0]
assert isinstance(data, Tensor), "Data should be a Tensor"
assert isinstance(label, Tensor), "Label should be a Tensor"
data, label = dataset[0]
assert isinstance(data, Tensor), "Data should be a Tensor"
assert isinstance(label, Tensor), "Label should be a Tensor"
assert data.shape == (5,), f"Data shape should be (5,), got {data.shape}"
assert label.shape == (), f"Label shape should be (), got {label.shape}"
print("✅ SimpleDataset sample access works correctly")
# Test sample shape
sample_shape = dataset.get_sample_shape()
sample_shape = dataset.get_sample_shape()
assert sample_shape == (5,), f"Sample shape should be (5,), got {sample_shape}"
print("✅ SimpleDataset get_sample_shape works correctly")
# Test multiple samples
for i in range(5):
data, label = dataset[i]
assert data.shape == (5,), f"Data shape should be (5,) for sample {i}, got {data.shape}"
assert 0 <= label.data < 4, f"Label should be in [0, 3] for sample {i}, got {label.data}"
data, label = dataset[i]
assert data.shape == (5,), f"Data shape should be (5,) for sample {i}, got {data.shape}"
assert 0 <= label.data < 4, f"Label should be in [0, 3] for sample {i}, got {label.data}"
print("✅ SimpleDataset multiple samples work correctly")
# Test deterministic data (same seed should give same data)
@@ -786,10 +786,9 @@ try:
assert np.array_equal(data1.data, data2.data), "Data should be deterministic"
assert np.array_equal(label1.data, label2.data), "Labels should be deterministic"
print("✅ SimpleDataset data is deterministic")
except Exception as e:
except Exception as e:
print(f"❌ SimpleDataset test failed: {e}")
raise
# Show the SimpleDataset behavior
print("🎯 SimpleDataset behavior:")
@@ -861,9 +860,9 @@ try:
# Verify batch properties
assert batch_data.shape[1] == 8, f"Features should be 8, got {batch_data.shape[1]}"
assert len(batch_labels.shape) == 1, f"Labels should be 1D, got shape {batch_labels.shape}"
assert isinstance(batch_data, Tensor), "Batch data should be Tensor"
assert isinstance(batch_labels, Tensor), "Batch labels should be Tensor"
assert isinstance(batch_data, Tensor), "Batch data should be Tensor"
assert isinstance(batch_labels, Tensor), "Batch labels should be Tensor"
assert epoch_samples == 100, f"Should process 100 samples, got {epoch_samples}"
expected_batches = (100 + 16 - 1) // 16
assert epoch_batches == expected_batches, f"Should have {expected_batches} batches, got {epoch_batches}"
@@ -887,8 +886,8 @@ try:
# Verify consistent batch processing
assert batch_data.shape[1] == 8, "Validation features should match training"
assert len(batch_labels.shape) == 1, "Validation labels should be 1D"
assert val_samples == 50, f"Should process 50 validation samples, got {val_samples}"
assert val_samples == 50, f"Should process 50 validation samples, got {val_samples}"
assert val_batches == 5, f"Should have 5 validation batches, got {val_batches}"
print("✅ Validation pipeline works correctly")
@@ -943,16 +942,16 @@ try:
dataset = SimpleDataset(size=60, num_features=6, num_classes=3)
loader = DataLoader(dataset, batch_size=20, shuffle=True)
for epoch in range(3):
epoch_samples = 0
for batch_data, batch_labels in loader:
epoch_samples += batch_data.shape[0]
# Verify shapes remain consistent across epochs
assert batch_data.shape[1] == 6, f"Features should be 6 in epoch {epoch}"
assert len(batch_labels.shape) == 1, f"Labels should be 1D in epoch {epoch}"
assert epoch_samples == 60, f"Should process 60 samples in epoch {epoch}, got {epoch_samples}"
for epoch in range(3):
epoch_samples = 0
for batch_data, batch_labels in loader:
epoch_samples += batch_data.shape[0]
# Verify shapes remain consistent across epochs
assert batch_data.shape[1] == 6, f"Features should be 6 in epoch {epoch}"
assert len(batch_labels.shape) == 1, f"Labels should be 1D in epoch {epoch}"
assert epoch_samples == 60, f"Should process 60 samples in epoch {epoch}, got {epoch_samples}"
print("✅ Multi-epoch training works correctly")