diff --git a/modules/source/08_dataloader/dataloader_dev.py b/modules/source/08_dataloader/dataloader_dev.py index 66afc6e2..df6921e6 100644 --- a/modules/source/08_dataloader/dataloader_dev.py +++ b/modules/source/08_dataloader/dataloader_dev.py @@ -43,6 +43,10 @@ import numpy as np import sys import os from typing import Tuple, Optional, Iterator +import urllib.request +import tarfile +import pickle +import time # Import our building blocks - try package first, then local modules try: @@ -710,6 +714,91 @@ class SimpleDataset(Dataset): """ return self.num_classes +# %% [markdown] +""" +## Step 4b: CIFAR-10 Dataset - Real Data for CNNs + +### Download and Load Real Computer Vision Data +Let's implement loading CIFAR-10, the dataset we'll use to achieve our north star goal of 75% accuracy! +""" + +# %% nbgrader={"grade": false, "grade_id": "cifar10", "locked": false, "schema_version": 3, "solution": true, "task": false} +#| export +def download_cifar10(root: str = "./data") -> str: + """ + Download CIFAR-10 dataset. + + TODO: Download and extract CIFAR-10. + + HINTS: + - URL: https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz + - Use urllib.request.urlretrieve() + - Extract with tarfile + """ + ### BEGIN SOLUTION + os.makedirs(root, exist_ok=True) + dataset_dir = os.path.join(root, "cifar-10-batches-py") + + if os.path.exists(dataset_dir): + print(f"โœ… CIFAR-10 found at {dataset_dir}") + return dataset_dir + + url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" + tar_path = os.path.join(root, "cifar-10.tar.gz") + + print(f"๐Ÿ“ฅ Downloading CIFAR-10 (~170MB)...") + urllib.request.urlretrieve(url, tar_path) + print("โœ… Downloaded!") + + print("๐Ÿ“ฆ Extracting...") + with tarfile.open(tar_path, 'r:gz') as tar: + tar.extractall(root) + print("โœ… Ready!") + + return dataset_dir + ### END SOLUTION + +class CIFAR10Dataset(Dataset): + """CIFAR-10 dataset for CNN training.""" + + def __init__(self, root="./data", train=True, download=False): + """Load CIFAR-10 data.""" + ### BEGIN SOLUTION + if download: + dataset_dir = download_cifar10(root) + else: + dataset_dir = os.path.join(root, "cifar-10-batches-py") + + if train: + data_list = [] + label_list = [] + for i in range(1, 6): + with open(os.path.join(dataset_dir, f"data_batch_{i}"), 'rb') as f: + batch = pickle.load(f, encoding='bytes') + data_list.append(batch[b'data']) + label_list.extend(batch[b'labels']) + self.data = np.concatenate(data_list) + self.labels = np.array(label_list) + else: + with open(os.path.join(dataset_dir, "test_batch"), 'rb') as f: + batch = pickle.load(f, encoding='bytes') + self.data = batch[b'data'] + self.labels = np.array(batch[b'labels']) + + # Reshape to (N, 3, 32, 32) and normalize + self.data = self.data.reshape(-1, 3, 32, 32).astype(np.float32) / 255.0 + print(f"โœ… Loaded {len(self.data):,} images") + ### END SOLUTION + + def __getitem__(self, idx): + return Tensor(self.data[idx]), Tensor(self.labels[idx]) + + def __len__(self): + return len(self.data) + + def get_num_classes(self): + return 10 + # %% [markdown] """ ### ๐Ÿงช Unit Test: SimpleDataset diff --git a/modules/source/11_training/training_dev.py b/modules/source/11_training/training_dev.py index 6f1ca926..84bada24 100644 --- a/modules/source/11_training/training_dev.py +++ b/modules/source/11_training/training_dev.py @@ -36,6 +36,7 @@ import sys import os from collections import defaultdict import time +import pickle # Add module directories to Python path sys.path.append(os.path.abspath('modules/source/02_tensor')) @@ -934,7 +935,7 @@ class Trainer: return epoch_metrics ### END SOLUTION - def fit(self, train_dataloader, val_dataloader=None, epochs=10, verbose=True): + def fit(self, train_dataloader, val_dataloader=None, epochs=10, verbose=True, save_best=False, checkpoint_path="best_model.pkl"): """ Train the model for specified number of epochs. @@ -971,6 +972,7 @@ class Trainer: """ ### BEGIN SOLUTION print(f"Starting training for {epochs} epochs...") + best_val_loss = float('inf') for epoch in range(epochs): self.current_epoch = epoch @@ -997,6 +999,14 @@ class Trainer: if val_dataloader is not None: self.history[f'val_{metric_name}'].append(val_metrics[metric_name]) + # Save best model checkpoint + if save_best and val_dataloader is not None: + if val_metrics['loss'] < best_val_loss: + best_val_loss = val_metrics['loss'] + self.save_checkpoint(checkpoint_path) + if verbose: + print(f" ๐Ÿ’พ Saved best model (val_loss: {best_val_loss:.4f})") + # Print progress if verbose: train_loss = train_metrics['loss'] @@ -1020,6 +1030,44 @@ class Trainer: print("Training completed!") return self.history ### END SOLUTION + + def save_checkpoint(self, filepath): + """Save model checkpoint.""" + checkpoint = { + 'epoch': self.current_epoch, + 'model_state': self._get_model_state(), + 'history': self.history + } + + with open(filepath, 'wb') as f: + pickle.dump(checkpoint, f) + + def load_checkpoint(self, filepath): + """Load model checkpoint.""" + with open(filepath, 'rb') as f: + checkpoint = pickle.load(f) + + self.current_epoch = checkpoint['epoch'] + self.history = checkpoint['history'] + self._set_model_state(checkpoint['model_state']) + + print(f"โœ… Loaded checkpoint from epoch {self.current_epoch}") + + def _get_model_state(self): + """Extract model parameters.""" + state = {} + for i, layer in enumerate(self.model.layers): + if hasattr(layer, 'weight'): + state[f'layer_{i}_weight'] = layer.weight.data.copy() + state[f'layer_{i}_bias'] = layer.bias.data.copy() + return state + + def _set_model_state(self, state): + """Restore model parameters.""" + for i, layer in enumerate(self.model.layers): + if hasattr(layer, 'weight'): + layer.weight.data = state[f'layer_{i}_weight'] + layer.bias.data = state[f'layer_{i}_bias'] # %% [markdown] """ @@ -1749,4 +1797,95 @@ if __name__ == "__main__": test_production_training_optimizer() print("All tests passed!") - print("training_dev module complete!") \ No newline at end of file + # Add evaluation tools for north star goal + print("training_dev module complete!") + +# %% [markdown] +""" +## Evaluation Tools for Model Analysis + +### Essential tools for achieving our north star goal of 75% CIFAR-10 accuracy +""" + +# %% nbgrader={"grade": false, "grade_id": "evaluation-tools", "locked": false, "schema_version": 3, "solution": true, "task": false} +#| export +def compute_confusion_matrix(model, dataloader, num_classes=10): + """ + Compute confusion matrix for classification model. + + Returns matrix where element (i,j) is count of samples with true label i predicted as j. + """ + ### BEGIN SOLUTION + confusion = np.zeros((num_classes, num_classes), dtype=int) + + for batch_x, batch_y in dataloader: + predictions = model(batch_x) + pred_labels = np.argmax(predictions.data, axis=1) + true_labels = batch_y.data.astype(int) + + for true, pred in zip(true_labels, pred_labels): + confusion[true, pred] += 1 + + return confusion + ### END SOLUTION + +def evaluate_model(model, dataloader): + """ + Evaluate model accuracy on dataset. + + Returns accuracy as percentage. + """ + ### BEGIN SOLUTION + correct = 0 + total = 0 + + for batch_x, batch_y in dataloader: + predictions = model(batch_x) + pred_labels = np.argmax(predictions.data, axis=1) + true_labels = batch_y.data.astype(int) + + correct += np.sum(pred_labels == true_labels) + total += len(true_labels) + + accuracy = (correct / total) * 100 + return accuracy + ### END SOLUTION + +def plot_training_history(history): + """ + Plot training curves (simplified for terminal output). + + Shows training and validation loss progression. + """ + ### BEGIN SOLUTION + if not history.get('epoch'): + print("No training history to plot") + return + + print("\n๐Ÿ“Š Training Progress:") + print("-" * 50) + + # Simple ASCII plot for loss + train_loss = history['train_loss'] + val_loss = history.get('val_loss', []) + + if train_loss: + min_loss = min(train_loss) + max_loss = max(train_loss) + + print(f"Train Loss: {train_loss[0]:.4f} โ†’ {train_loss[-1]:.4f}") + if val_loss: + print(f"Val Loss: {val_loss[0]:.4f} โ†’ {val_loss[-1]:.4f}") + + # Show trend + if train_loss[-1] < train_loss[0]: + print("โœ… Training loss decreased (model is learning!)") + else: + print("โš ๏ธ Training loss increased (check learning rate)") + + if val_loss and len(val_loss) > 1: + if val_loss[-1] > val_loss[-2]: + print("โš ๏ธ Validation loss increasing (possible overfitting)") + + print("-" * 50) + ### END SOLUTION \ No newline at end of file