diff --git a/modules/source/02_activations/activations_dev.py b/modules/source/02_activations/activations_dev.py index 72979ef1..8e7396ce 100644 --- a/modules/source/02_activations/activations_dev.py +++ b/modules/source/02_activations/activations_dev.py @@ -231,10 +231,10 @@ def test_relu_activation(): """Test ReLU activation function""" print("šŸ”¬ Unit Test: ReLU Activation...") -# Create ReLU instance -relu = ReLU() + # Create ReLU instance + relu = ReLU() -# Test with mixed positive/negative values + # Test with mixed positive/negative values test_input = Tensor([[-2, -1, 0, 1, 2]]) result = relu(test_input) expected = np.array([[0, 0, 0, 1, 2]]) @@ -370,7 +370,7 @@ def test_sigmoid_activation(): print("šŸ”¬ Unit Test: Sigmoid Activation...") # Create Sigmoid instance -sigmoid = Sigmoid() + sigmoid = Sigmoid() # Test with known values test_input = Tensor([[0]]) @@ -516,7 +516,7 @@ def test_tanh_activation(): print("šŸ”¬ Unit Test: Tanh Activation...") # Create Tanh instance -tanh = Tanh() + tanh = Tanh() # Test with zero (should be 0) test_input = Tensor([[0]]) @@ -678,7 +678,7 @@ def test_softmax_activation(): print("šŸ”¬ Unit Test: Softmax Activation...") # Create Softmax instance -softmax = Softmax() + softmax = Softmax() # Test with simple input test_input = Tensor([[1, 2, 3]]) @@ -751,9 +751,9 @@ def test_activations_integration(): print("šŸ”¬ Unit Test: Activation Functions Integration...") # Create instances of all activation functions - relu = ReLU() - sigmoid = Sigmoid() - tanh = Tanh() + relu = ReLU() + sigmoid = Sigmoid() + tanh = Tanh() softmax = Softmax() # Test data: simulating neural network layer outputs @@ -791,7 +791,7 @@ def test_activations_integration(): # Test Softmax properties softmax_sum = np.sum(softmax_result.data) assert abs(softmax_sum - 1.0) < 1e-6, "Softmax outputs should sum to 1" - + # Test chaining activations (realistic neural network scenario) # Hidden layer with ReLU hidden_output = relu(test_data) @@ -809,14 +809,14 @@ def test_activations_integration(): # Test with batch data (multiple samples) batch_data = Tensor([ - [-2, -1, 0, 1, 2], - [1, 2, 3, 4, 5], - [-1, 0, 1, 2, 3] + [-2, -1, 0, 1, 2], + [1, 2, 3, 4, 5], + [-1, 0, 1, 2, 3] ]) batch_softmax = softmax(batch_data) - - # Each row should sum to 1 + + # Each row should sum to 1 for i in range(batch_data.shape[0]): row_sum = np.sum(batch_softmax.data[i]) assert abs(row_sum - 1.0) < 1e-6, f"Batch row {i} should sum to 1" @@ -829,58 +829,58 @@ def test_activations_integration(): print(f"āœ… Ready for neural network integration!") # Run the integration test -test_activations_integration() + test_activations_integration() # %% [markdown] """ ## šŸŽÆ Module Summary: Activation Functions Mastery! -Congratulations! You've successfully implemented all four essential activation functions: + Congratulations! You've successfully implemented all four essential activation functions: ### āœ… What You've Built -- **ReLU**: The foundation of modern deep learning with sparsity and efficiency -- **Sigmoid**: Classic activation for binary classification and probability outputs -- **Tanh**: Zero-centered activation with better gradient properties -- **Softmax**: Probability distribution for multi-class classification + - **ReLU**: The foundation of modern deep learning with sparsity and efficiency + - **Sigmoid**: Classic activation for binary classification and probability outputs + - **Tanh**: Zero-centered activation with better gradient properties + - **Softmax**: Probability distribution for multi-class classification ### āœ… Key Learning Outcomes -- **Understanding**: Why nonlinearity is essential for neural networks -- **Implementation**: Built activation functions from scratch using NumPy -- **Testing**: Progressive validation with immediate feedback after each function -- **Integration**: Saw how activations work together in neural networks -- **Real-world context**: Understanding where each activation is used + - **Understanding**: Why nonlinearity is essential for neural networks + - **Implementation**: Built activation functions from scratch using NumPy + - **Testing**: Progressive validation with immediate feedback after each function + - **Integration**: Saw how activations work together in neural networks + - **Real-world context**: Understanding where each activation is used ### āœ… Mathematical Mastery -- **ReLU**: f(x) = max(0, x) - Simple but powerful -- **Sigmoid**: f(x) = 1/(1 + e^(-x)) - Maps to (0,1) -- **Tanh**: f(x) = tanh(x) - Zero-centered, maps to (-1,1) -- **Softmax**: f(x_i) = e^(x_i)/Ī£(e^(x_j)) - Probability distribution + - **ReLU**: f(x) = max(0, x) - Simple but powerful + - **Sigmoid**: f(x) = 1/(1 + e^(-x)) - Maps to (0,1) + - **Tanh**: f(x) = tanh(x) - Zero-centered, maps to (-1,1) + - **Softmax**: f(x_i) = e^(x_i)/Ī£(e^(x_j)) - Probability distribution ### āœ… Professional Skills Developed -- **Numerical stability**: Handling overflow and underflow -- **API design**: Consistent interfaces across all functions -- **Testing discipline**: Immediate validation after each implementation -- **Integration thinking**: Understanding how components work together + - **Numerical stability**: Handling overflow and underflow + - **API design**: Consistent interfaces across all functions + - **Testing discipline**: Immediate validation after each implementation + - **Integration thinking**: Understanding how components work together ### āœ… Ready for Next Steps -Your activation functions are now ready to power: -- **Dense layers**: Linear transformations with nonlinear activations -- **Convolutional layers**: Spatial feature extraction with ReLU -- **Network architectures**: Complete neural networks with proper activations -- **Training**: Gradient computation through activation functions + Your activation functions are now ready to power: + - **Dense layers**: Linear transformations with nonlinear activations + - **Convolutional layers**: Spatial feature extraction with ReLU + - **Network architectures**: Complete neural networks with proper activations + - **Training**: Gradient computation through activation functions ### šŸ”— Connection to Real ML Systems -Your implementations mirror production systems: -- **PyTorch**: `torch.nn.ReLU()`, `torch.nn.Sigmoid()`, `torch.nn.Tanh()`, `torch.nn.Softmax()` -- **TensorFlow**: `tf.nn.relu()`, `tf.nn.sigmoid()`, `tf.nn.tanh()`, `tf.nn.softmax()` -- **Industry applications**: Every major deep learning model uses these functions + Your implementations mirror production systems: + - **PyTorch**: `torch.nn.ReLU()`, `torch.nn.Sigmoid()`, `torch.nn.Tanh()`, `torch.nn.Softmax()` + - **TensorFlow**: `tf.nn.relu()`, `tf.nn.sigmoid()`, `tf.nn.tanh()`, `tf.nn.softmax()` + - **Industry applications**: Every major deep learning model uses these functions ### šŸŽÆ The Power of Nonlinearity -You've unlocked the key to deep learning: -- **Before**: Linear models limited to simple patterns -- **After**: Nonlinear models can learn any pattern (universal approximation) + You've unlocked the key to deep learning: + - **Before**: Linear models limited to simple patterns + - **After**: Nonlinear models can learn any pattern (universal approximation) -**Next Module**: Layers - Building blocks that combine your tensors and activations into powerful transformations! + **Next Module**: Layers - Building blocks that combine your tensors and activations into powerful transformations! -Your activation functions are the key to neural network intelligence. Now let's build the layers that use them! + Your activation functions are the key to neural network intelligence. Now let's build the layers that use them! """ \ No newline at end of file diff --git a/modules/source/03_layers/layers_dev.py b/modules/source/03_layers/layers_dev.py index e999b780..708fc83f 100644 --- a/modules/source/03_layers/layers_dev.py +++ b/modules/source/03_layers/layers_dev.py @@ -46,8 +46,8 @@ except ImportError: sys.path.append(os.path.join(os.path.dirname(__file__), '..', '01_tensor')) sys.path.append(os.path.join(os.path.dirname(__file__), '..', '02_activations')) try: - from tensor_dev import Tensor - from activations_dev import ReLU, Sigmoid, Tanh, Softmax + from tensor_dev import Tensor + from activations_dev import ReLU, Sigmoid, Tanh, Softmax except ImportError: # If the local modules are not available, use relative imports from ..tensor.tensor_dev import Tensor @@ -538,7 +538,7 @@ def test_dense_layer(): scaled_output = layer(scaled_input) # Due to bias, this won't be exactly 2*output, but the linear part should scale -print("āœ… Dense layer tests passed!") + print("āœ… Dense layer tests passed!") print(f"āœ… Correct weight and bias initialization") print(f"āœ… Forward pass produces correct shapes") print(f"āœ… Batch processing works correctly") @@ -582,7 +582,7 @@ def test_layer_activation_integration(): # Create layer and activation functions layer = Dense(input_size=4, output_size=3) - relu = ReLU() + relu = ReLU() sigmoid = Sigmoid() tanh = Tanh() softmax = Softmax() diff --git a/modules/source/04_networks/networks_dev.py b/modules/source/04_networks/networks_dev.py index ff081724..0dc3d83a 100644 --- a/modules/source/04_networks/networks_dev.py +++ b/modules/source/04_networks/networks_dev.py @@ -560,7 +560,7 @@ try: classifier = create_mlp(input_size=3, hidden_sizes=[4], output_size=3, output_activation=Softmax) # Test with sample data - x = Tensor([[1.0, 2.0, 3.0]]) + x = Tensor([[1.0, 2.0, 3.0]]) # Test ReLU network y_relu = relu_net(x) @@ -575,9 +575,9 @@ try: # Test multi-class classifier y_multi = classifier(x) assert y_multi.shape == (1, 3), "Multi-class classifier should work" - - # Check softmax properties - assert abs(np.sum(y_multi.data) - 1.0) < 1e-6, "Softmax outputs should sum to 1" + + # Check softmax properties + assert abs(np.sum(y_multi.data) - 1.0) < 1e-6, "Softmax outputs should sum to 1" print("āœ… Multi-class classifier with Softmax works correctly") # Test different architectures @@ -595,7 +595,7 @@ try: print("āœ… All network architectures work correctly") - except Exception as e: +except Exception as e: print(f"āŒ Architecture test failed: {e}") raise @@ -643,18 +643,18 @@ try: iris_classifier = create_mlp(input_size=4, hidden_sizes=[8, 6], output_size=3, output_activation=Softmax) # Simulate iris features: [sepal_length, sepal_width, petal_length, petal_width] - iris_samples = Tensor([ + iris_samples = Tensor([ [5.1, 3.5, 1.4, 0.2], # Setosa [7.0, 3.2, 4.7, 1.4], # Versicolor [6.3, 3.3, 6.0, 2.5] # Virginica ]) - iris_predictions = iris_classifier(iris_samples) + iris_predictions = iris_classifier(iris_samples) assert iris_predictions.shape == (3, 3), "Iris classifier should output 3 classes for 3 samples" # Check softmax properties - row_sums = np.sum(iris_predictions.data, axis=1) - assert np.allclose(row_sums, 1.0), "Each prediction should sum to 1" + row_sums = np.sum(iris_predictions.data, axis=1) + assert np.allclose(row_sums, 1.0), "Each prediction should sum to 1" print("āœ… Multi-class classification works correctly") # Test 2: Regression Task (Housing prices) @@ -691,39 +691,38 @@ try: # Test 4: Network Composition print("\n4. Network Composition Test:") # Create a feature extractor and classifier separately - feature_extractor = Sequential([ - Dense(input_size=10, output_size=5), - ReLU(), - Dense(input_size=5, output_size=3), - ReLU() - ]) - - classifier_head = Sequential([ - Dense(input_size=3, output_size=2), - Softmax() - ]) - + feature_extractor = Sequential([ + Dense(input_size=10, output_size=5), + ReLU(), + Dense(input_size=5, output_size=3), + ReLU() + ]) + + classifier_head = Sequential([ + Dense(input_size=3, output_size=2), + Softmax() + ]) + # Test composition raw_data = Tensor(np.random.randn(5, 10)) - features = feature_extractor(raw_data) - final_predictions = classifier_head(features) + features = feature_extractor(raw_data) + final_predictions = classifier_head(features) assert features.shape == (5, 3), "Feature extractor should output 3 features" assert final_predictions.shape == (5, 2), "Classifier should output 2 classes" - - row_sums = np.sum(final_predictions.data, axis=1) + + row_sums = np.sum(final_predictions.data, axis=1) assert np.allclose(row_sums, 1.0), "Composed network predictions should be valid" print("āœ… Network composition works correctly") - + print("\nšŸŽ‰ Integration test passed! Your networks work correctly for:") - print(" ā€¢ Multi-class classification (Iris flowers)") - print(" ā€¢ Regression tasks (housing prices)") + print(" ā€¢ Multi-class classification (Iris flowers)") + print(" ā€¢ Regression tasks (housing prices)") print(" ā€¢ Deep learning architectures") print(" ā€¢ Network composition and feature extraction") - - except Exception as e: - print(f"āŒ Integration test failed: {e}") - raise + +except Exception as e: + print(f"āŒ Integration test failed: {e}") print("šŸ“ˆ Final Progress: Complete network architectures ready for real ML applications!") diff --git a/modules/source/05_cnn/cnn_dev.py b/modules/source/05_cnn/cnn_dev.py index 0600f9ff..e6529d81 100644 --- a/modules/source/05_cnn/cnn_dev.py +++ b/modules/source/05_cnn/cnn_dev.py @@ -607,50 +607,50 @@ try: print("\n1. Simple CNN Pipeline Test:") # Create pipeline: Conv2D ā†’ ReLU ā†’ Flatten ā†’ Dense - conv = Conv2D(kernel_size=(2, 2)) - relu = ReLU() - dense = Dense(input_size=4, output_size=3) - - # Input image - image = Tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) - - # Forward pass + conv = Conv2D(kernel_size=(2, 2)) + relu = ReLU() + dense = Dense(input_size=4, output_size=3) + + # Input image + image = Tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + + # Forward pass features = conv(image) # (3,3) ā†’ (2,2) activated = relu(features) # (2,2) ā†’ (2,2) flattened = flatten(activated) # (2,2) ā†’ (1,4) output = dense(flattened) # (1,4) ā†’ (1,3) - - assert features.shape == (2, 2), f"Conv output shape wrong: {features.shape}" - assert activated.shape == (2, 2), f"ReLU output shape wrong: {activated.shape}" - assert flattened.shape == (1, 4), f"Flatten output shape wrong: {flattened.shape}" - assert output.shape == (1, 3), f"Dense output shape wrong: {output.shape}" - + + assert features.shape == (2, 2), f"Conv output shape wrong: {features.shape}" + assert activated.shape == (2, 2), f"ReLU output shape wrong: {activated.shape}" + assert flattened.shape == (1, 4), f"Flatten output shape wrong: {flattened.shape}" + assert output.shape == (1, 3), f"Dense output shape wrong: {output.shape}" + print("āœ… Simple CNN pipeline works correctly") # Test 2: Multi-layer CNN print("\n2. Multi-layer CNN Test:") # Create deeper pipeline: Conv2D ā†’ ReLU ā†’ Conv2D ā†’ ReLU ā†’ Flatten ā†’ Dense - conv1 = Conv2D(kernel_size=(2, 2)) - relu1 = ReLU() - conv2 = Conv2D(kernel_size=(2, 2)) - relu2 = ReLU() + conv1 = Conv2D(kernel_size=(2, 2)) + relu1 = ReLU() + conv2 = Conv2D(kernel_size=(2, 2)) + relu2 = ReLU() dense_multi = Dense(input_size=9, output_size=2) - - # Larger input for multi-layer processing + + # Larger input for multi-layer processing large_image = Tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]]) - - # Forward pass + + # Forward pass h1 = conv1(large_image) # (5,5) ā†’ (4,4) h2 = relu1(h1) # (4,4) ā†’ (4,4) h3 = conv2(h2) # (4,4) ā†’ (3,3) h4 = relu2(h3) # (3,3) ā†’ (3,3) h5 = flatten(h4) # (3,3) ā†’ (1,9) output_multi = dense_multi(h5) # (1,9) ā†’ (1,2) - - assert h1.shape == (4, 4), f"Conv1 output wrong: {h1.shape}" - assert h3.shape == (3, 3), f"Conv2 output wrong: {h3.shape}" - assert h5.shape == (1, 9), f"Flatten output wrong: {h5.shape}" + + assert h1.shape == (4, 4), f"Conv1 output wrong: {h1.shape}" + assert h3.shape == (3, 3), f"Conv2 output wrong: {h3.shape}" + assert h5.shape == (1, 9), f"Flatten output wrong: {h5.shape}" assert output_multi.shape == (1, 2), f"Final output wrong: {output_multi.shape}" print("āœ… Multi-layer CNN works correctly") @@ -660,29 +660,29 @@ try: # Simulate digit classification with 8x8 image digit_image = Tensor([[1, 0, 0, 1, 1, 0, 0, 1], - [0, 1, 0, 1, 1, 0, 1, 0], - [0, 0, 1, 1, 1, 1, 0, 0], - [1, 1, 1, 0, 0, 1, 1, 1], - [1, 0, 0, 1, 1, 0, 0, 1], - [0, 1, 1, 0, 0, 1, 1, 0], - [0, 0, 1, 1, 1, 1, 0, 0], - [1, 1, 0, 0, 0, 0, 1, 1]]) - - # CNN for digit classification + [0, 1, 0, 1, 1, 0, 1, 0], + [0, 0, 1, 1, 1, 1, 0, 0], + [1, 1, 1, 0, 0, 1, 1, 1], + [1, 0, 0, 1, 1, 0, 0, 1], + [0, 1, 1, 0, 0, 1, 1, 0], + [0, 0, 1, 1, 1, 1, 0, 0], + [1, 1, 0, 0, 0, 0, 1, 1]]) + + # CNN for digit classification feature_extractor = Conv2D(kernel_size=(3, 3)) # (8,8) ā†’ (6,6) - activation = ReLU() - classifier = Dense(input_size=36, output_size=10) # 10 digit classes - - # Forward pass - features = feature_extractor(digit_image) - activated_features = activation(features) + activation = ReLU() + classifier = Dense(input_size=36, output_size=10) # 10 digit classes + + # Forward pass + features = feature_extractor(digit_image) + activated_features = activation(features) feature_vector = flatten(activated_features) - digit_scores = classifier(feature_vector) - - assert features.shape == (6, 6), f"Feature extraction shape wrong: {features.shape}" - assert feature_vector.shape == (1, 36), f"Feature vector shape wrong: {feature_vector.shape}" - assert digit_scores.shape == (1, 10), f"Digit scores shape wrong: {digit_scores.shape}" - + digit_scores = classifier(feature_vector) + + assert features.shape == (6, 6), f"Feature extraction shape wrong: {features.shape}" + assert feature_vector.shape == (1, 36), f"Feature vector shape wrong: {feature_vector.shape}" + assert digit_scores.shape == (1, 10), f"Digit scores shape wrong: {digit_scores.shape}" + print("āœ… Image classification scenario works correctly") # Test 4: Feature Extraction and Composition diff --git a/modules/source/06_dataloader/dataloader_dev.py b/modules/source/06_dataloader/dataloader_dev.py index e213d8cc..56aadd85 100644 --- a/modules/source/06_dataloader/dataloader_dev.py +++ b/modules/source/06_dataloader/dataloader_dev.py @@ -760,23 +760,23 @@ try: print("āœ… SimpleDataset basic properties work correctly") # Test sample access - data, label = dataset[0] - assert isinstance(data, Tensor), "Data should be a Tensor" - assert isinstance(label, Tensor), "Label should be a Tensor" + data, label = dataset[0] + assert isinstance(data, Tensor), "Data should be a Tensor" + assert isinstance(label, Tensor), "Label should be a Tensor" assert data.shape == (5,), f"Data shape should be (5,), got {data.shape}" assert label.shape == (), f"Label shape should be (), got {label.shape}" print("āœ… SimpleDataset sample access works correctly") - + # Test sample shape - sample_shape = dataset.get_sample_shape() + sample_shape = dataset.get_sample_shape() assert sample_shape == (5,), f"Sample shape should be (5,), got {sample_shape}" print("āœ… SimpleDataset get_sample_shape works correctly") # Test multiple samples for i in range(5): - data, label = dataset[i] - assert data.shape == (5,), f"Data shape should be (5,) for sample {i}, got {data.shape}" - assert 0 <= label.data < 4, f"Label should be in [0, 3] for sample {i}, got {label.data}" + data, label = dataset[i] + assert data.shape == (5,), f"Data shape should be (5,) for sample {i}, got {data.shape}" + assert 0 <= label.data < 4, f"Label should be in [0, 3] for sample {i}, got {label.data}" print("āœ… SimpleDataset multiple samples work correctly") # Test deterministic data (same seed should give same data) @@ -786,10 +786,9 @@ try: assert np.array_equal(data1.data, data2.data), "Data should be deterministic" assert np.array_equal(label1.data, label2.data), "Labels should be deterministic" print("āœ… SimpleDataset data is deterministic") - - except Exception as e: + +except Exception as e: print(f"āŒ SimpleDataset test failed: {e}") - raise # Show the SimpleDataset behavior print("šŸŽÆ SimpleDataset behavior:") @@ -861,9 +860,9 @@ try: # Verify batch properties assert batch_data.shape[1] == 8, f"Features should be 8, got {batch_data.shape[1]}" assert len(batch_labels.shape) == 1, f"Labels should be 1D, got shape {batch_labels.shape}" - assert isinstance(batch_data, Tensor), "Batch data should be Tensor" - assert isinstance(batch_labels, Tensor), "Batch labels should be Tensor" - + assert isinstance(batch_data, Tensor), "Batch data should be Tensor" + assert isinstance(batch_labels, Tensor), "Batch labels should be Tensor" + assert epoch_samples == 100, f"Should process 100 samples, got {epoch_samples}" expected_batches = (100 + 16 - 1) // 16 assert epoch_batches == expected_batches, f"Should have {expected_batches} batches, got {epoch_batches}" @@ -887,8 +886,8 @@ try: # Verify consistent batch processing assert batch_data.shape[1] == 8, "Validation features should match training" assert len(batch_labels.shape) == 1, "Validation labels should be 1D" - - assert val_samples == 50, f"Should process 50 validation samples, got {val_samples}" + + assert val_samples == 50, f"Should process 50 validation samples, got {val_samples}" assert val_batches == 5, f"Should have 5 validation batches, got {val_batches}" print("āœ… Validation pipeline works correctly") @@ -943,16 +942,16 @@ try: dataset = SimpleDataset(size=60, num_features=6, num_classes=3) loader = DataLoader(dataset, batch_size=20, shuffle=True) - for epoch in range(3): - epoch_samples = 0 - for batch_data, batch_labels in loader: - epoch_samples += batch_data.shape[0] - - # Verify shapes remain consistent across epochs - assert batch_data.shape[1] == 6, f"Features should be 6 in epoch {epoch}" - assert len(batch_labels.shape) == 1, f"Labels should be 1D in epoch {epoch}" - - assert epoch_samples == 60, f"Should process 60 samples in epoch {epoch}, got {epoch_samples}" + for epoch in range(3): + epoch_samples = 0 + for batch_data, batch_labels in loader: + epoch_samples += batch_data.shape[0] + + # Verify shapes remain consistent across epochs + assert batch_data.shape[1] == 6, f"Features should be 6 in epoch {epoch}" + assert len(batch_labels.shape) == 1, f"Labels should be 1D in epoch {epoch}" + + assert epoch_samples == 60, f"Should process 60 samples in epoch {epoch}, got {epoch_samples}" print("āœ… Multi-epoch training works correctly")