mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-05-01 06:48:52 -05:00
Add tito grade command for simplified NBGrader interface
Implement comprehensive grading workflow wrapped behind tito CLI: • tito grade setup - Initialize NBGrader course structure • tito grade generate - Create instructor version with solutions • tito grade release - Create student version without solutions • tito grade collect - Collect student submissions • tito grade autograde - Automatically grade submissions • tito grade manual - Open manual grading interface • tito grade feedback - Generate student feedback • tito grade export - Export grades to CSV This allows users to only learn tito commands without needing to understand NBGrader's complex interface. All grading functionality is accessible through simple, consistent tito commands.
This commit is contained in:
Vijay Janapa Reddi
@@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a244993b",
|
||||
"id": "40c65585",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\""
|
||||
},
|
||||
@@ -27,7 +27,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e007e4c9",
|
||||
"id": "31270c93",
|
||||
"metadata": {
|
||||
"nbgrader": {
|
||||
"grade": false,
|
||||
@@ -79,7 +79,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4a7b5fce",
|
||||
"id": "6e8e28fe",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\""
|
||||
},
|
||||
@@ -89,7 +89,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e893e77f",
|
||||
"id": "c9131846",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -150,7 +150,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3888f0a5",
|
||||
"id": "c5d8a8d0",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -228,7 +228,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "14c59bb3",
|
||||
"id": "b92c8aec",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -242,7 +242,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "dc01fcb9",
|
||||
"id": "0c53032b",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -301,7 +301,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1b2cd6b4",
|
||||
"id": "2aee7ae1",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -405,7 +405,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "04480fab",
|
||||
"id": "7f25886d",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -419,7 +419,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "061aa165",
|
||||
"id": "c3507f41",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -476,7 +476,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "072cb810",
|
||||
"id": "1fc153fc",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -585,7 +585,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "72fd1695",
|
||||
"id": "67acce3f",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -599,7 +599,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aa764594",
|
||||
"id": "d0b63b9d",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -655,7 +655,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c5b978ab",
|
||||
"id": "1861dbea",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"\"\"\"\n",
|
||||
@@ -709,7 +709,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ac9edeba",
|
||||
"id": "90622dc6",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -806,7 +806,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "637974f0",
|
||||
"id": "8a896363",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -820,7 +820,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "607252fc",
|
||||
"id": "da3306d7",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -876,7 +876,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ff8bc0e5",
|
||||
"id": "18f049c2",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -923,7 +923,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bef96832",
|
||||
"id": "560a2d73",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1283,7 +1283,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c49fcceb",
|
||||
"id": "d5af843c",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -1297,7 +1297,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4ed5911e",
|
||||
"id": "432f193e",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1350,7 +1350,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e1e1f9ce",
|
||||
"id": "79f7e6b4",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -1366,7 +1366,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c2c7e875",
|
||||
"id": "8e40d9cf",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1453,7 +1453,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b992bc0f",
|
||||
"id": "554a2102",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -1486,7 +1486,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "054beb05",
|
||||
"id": "66602940",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1689,7 +1689,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "57a786fa",
|
||||
"id": "f136e224",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -1703,7 +1703,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4499cc18",
|
||||
"id": "232e80c5",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1774,7 +1774,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f5f911db",
|
||||
"id": "16381eea",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
@@ -1950,7 +1950,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ac1b6d8b",
|
||||
"id": "cc2f03ca",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
@@ -1964,9 +1964,8 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6e440542",
|
||||
"id": "81d4dc22",
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 1,
|
||||
"nbgrader": {
|
||||
"grade": false,
|
||||
"grade_id": "test-production-optimizer",
|
||||
@@ -2033,12 +2032,24 @@
|
||||
" \n",
|
||||
" print(\"🎯 Production Training Optimizer: All tests passed!\")\n",
|
||||
"\n",
|
||||
"# Test function defined (called in main block)"
|
||||
"# Test function defined (called in main block)\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" # Run all training tests\n",
|
||||
" test_unit_simple_training_loop()\n",
|
||||
" test_unit_batch_training()\n",
|
||||
" test_unit_multiple_epochs()\n",
|
||||
" test_unit_training_with_validation()\n",
|
||||
" test_module_training_pipeline_integration()\n",
|
||||
" test_training_pipeline_profiler()\n",
|
||||
" \n",
|
||||
" print(\"All tests passed!\")\n",
|
||||
" print(\"Training module complete!\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0a1a5f6d",
|
||||
"id": "29e6d19b",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\""
|
||||
},
|
||||
@@ -2080,15 +2091,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "022a9f18",
|
||||
"metadata": {},
|
||||
"id": "ec2e9914",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\""
|
||||
},
|
||||
"source": [
|
||||
"\"\"\"\n",
|
||||
"# 🎯 MODULE SUMMARY: Training Pipelines\n",
|
||||
"## 🎯 MODULE SUMMARY: Training Pipelines\n",
|
||||
"\n",
|
||||
"Congratulations! You've successfully implemented complete training pipelines:\n",
|
||||
"\n",
|
||||
"## What You've Accomplished\n",
|
||||
"### What You've Accomplished\n",
|
||||
"✅ **Training Loops**: End-to-end training with loss computation and optimization \n",
|
||||
"✅ **Loss Functions**: Implementation and integration of loss calculations \n",
|
||||
"✅ **Metrics Tracking**: Monitoring accuracy and loss during training \n",
|
||||
@@ -2097,7 +2109,7 @@
|
||||
"✅ **Pipeline Profiling**: Production-grade performance analysis and optimization \n",
|
||||
"✅ **Systems Thinking**: Understanding training infrastructure at scale \n",
|
||||
"\n",
|
||||
"## Key Concepts You've Learned\n",
|
||||
"### Key Concepts You've Learned\n",
|
||||
"- **Training loops**: How to iterate over data, compute loss, and update parameters\n",
|
||||
"- **Loss functions**: Quantifying model performance\n",
|
||||
"- **Metrics tracking**: Monitoring progress and diagnosing issues\n",
|
||||
@@ -2106,7 +2118,7 @@
|
||||
"- **Pipeline profiling**: Identifying bottlenecks in training infrastructure\n",
|
||||
"- **Production optimization**: Balancing throughput, memory, and resource utilization\n",
|
||||
"\n",
|
||||
"## Professional Skills Developed\n",
|
||||
"### Professional Skills Developed\n",
|
||||
"- **Training orchestration**: Building robust training systems\n",
|
||||
"- **Loss engineering**: Implementing and tuning loss functions\n",
|
||||
"- **Metrics analysis**: Understanding and improving model performance\n",
|
||||
@@ -2114,7 +2126,7 @@
|
||||
"- **Performance profiling**: Optimizing training pipelines for production\n",
|
||||
"- **Systems design**: Understanding distributed training challenges\n",
|
||||
"\n",
|
||||
"## Ready for Advanced Applications\n",
|
||||
"### Ready for Advanced Applications\n",
|
||||
"Your training pipeline implementations now enable:\n",
|
||||
"- **Full model training**: End-to-end training of neural networks\n",
|
||||
"- **Experimentation**: Testing different architectures and hyperparameters\n",
|
||||
@@ -2123,148 +2135,20 @@
|
||||
"- **Performance optimization**: Scaling training to production workloads\n",
|
||||
"- **Infrastructure design**: Building reliable ML training systems\n",
|
||||
"\n",
|
||||
"## Connection to Real ML Systems\n",
|
||||
"### Connection to Real ML Systems\n",
|
||||
"Your implementations mirror production systems:\n",
|
||||
"- **PyTorch**: `torch.nn.Module`, `torch.optim`, and training loops\n",
|
||||
"- **TensorFlow**: `tf.keras.Model`, `tf.keras.optimizers`, and fit methods\n",
|
||||
"- **Industry Standard**: Every major ML framework uses these exact patterns\n",
|
||||
"- **Production Tools**: Similar to Ray Train, Horovod, and distributed training frameworks\n",
|
||||
"\n",
|
||||
"## Next Steps\n",
|
||||
"### Next Steps\n",
|
||||
"1. **Export your code**: `tito export 11_training`\n",
|
||||
"2. **Test your implementation**: `tito test 11_training`\n",
|
||||
"3. **Build evaluation pipelines**: Add benchmarking and validation\n",
|
||||
"4. **Move to Module 12**: Add model compression and optimization!\n",
|
||||
"\n",
|
||||
"**Ready for compression?** Your training pipelines are now ready for real-world deployment!\n",
|
||||
"\"\"\" \n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" # Run all tests\n",
|
||||
" test_unit_mse_loss()\n",
|
||||
" test_unit_crossentropy_loss()\n",
|
||||
" test_unit_binary_crossentropy_loss()\n",
|
||||
" test_unit_accuracy_metric()\n",
|
||||
" test_unit_trainer()\n",
|
||||
" test_module_training()\n",
|
||||
" test_training_pipeline_profiler()\n",
|
||||
" test_production_training_optimizer()\n",
|
||||
" \n",
|
||||
" print(\"All tests passed!\")\n",
|
||||
" # Add evaluation tools for north star goal\n",
|
||||
" print(\"training_dev module complete!\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "857dfdb5",
|
||||
"metadata": {
|
||||
"cell_marker": "\"\"\"",
|
||||
"lines_to_next_cell": 1
|
||||
},
|
||||
"source": [
|
||||
"## Evaluation Tools for Model Analysis\n",
|
||||
"\n",
|
||||
"### Essential tools for achieving our north star goal of 75% CIFAR-10 accuracy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b52a383b",
|
||||
"metadata": {
|
||||
"nbgrader": {
|
||||
"grade": false,
|
||||
"grade_id": "evaluation-tools",
|
||||
"locked": false,
|
||||
"schema_version": 3,
|
||||
"solution": true,
|
||||
"task": false
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#| export\n",
|
||||
"def compute_confusion_matrix(model, dataloader, num_classes=10):\n",
|
||||
" \"\"\"\n",
|
||||
" Compute confusion matrix for classification model.\n",
|
||||
" \n",
|
||||
" Returns matrix where element (i,j) is count of samples with true label i predicted as j.\n",
|
||||
" \"\"\"\n",
|
||||
" ### BEGIN SOLUTION\n",
|
||||
" confusion = np.zeros((num_classes, num_classes), dtype=int)\n",
|
||||
" \n",
|
||||
" for batch_x, batch_y in dataloader:\n",
|
||||
" predictions = model(batch_x)\n",
|
||||
" pred_labels = np.argmax(predictions.data, axis=1)\n",
|
||||
" true_labels = batch_y.data.astype(int)\n",
|
||||
" \n",
|
||||
" for true, pred in zip(true_labels, pred_labels):\n",
|
||||
" confusion[true, pred] += 1\n",
|
||||
" \n",
|
||||
" return confusion\n",
|
||||
" ### END SOLUTION\n",
|
||||
"\n",
|
||||
"def evaluate_model(model, dataloader):\n",
|
||||
" \"\"\"\n",
|
||||
" Evaluate model accuracy on dataset.\n",
|
||||
" \n",
|
||||
" Returns accuracy as percentage.\n",
|
||||
" \"\"\"\n",
|
||||
" ### BEGIN SOLUTION\n",
|
||||
" correct = 0\n",
|
||||
" total = 0\n",
|
||||
" \n",
|
||||
" for batch_x, batch_y in dataloader:\n",
|
||||
" predictions = model(batch_x)\n",
|
||||
" pred_labels = np.argmax(predictions.data, axis=1)\n",
|
||||
" true_labels = batch_y.data.astype(int)\n",
|
||||
" \n",
|
||||
" correct += np.sum(pred_labels == true_labels)\n",
|
||||
" total += len(true_labels)\n",
|
||||
" \n",
|
||||
" accuracy = (correct / total) * 100\n",
|
||||
" return accuracy\n",
|
||||
" ### END SOLUTION\n",
|
||||
"\n",
|
||||
"def plot_training_history(history):\n",
|
||||
" \"\"\"\n",
|
||||
" Plot training curves (simplified for terminal output).\n",
|
||||
" \n",
|
||||
" Shows training and validation loss progression.\n",
|
||||
" \"\"\"\n",
|
||||
" ### BEGIN SOLUTION\n",
|
||||
" if not history.get('epoch'):\n",
|
||||
" print(\"No training history to plot\")\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" print(\"\\n📊 Training Progress:\")\n",
|
||||
" print(\"-\" * 50)\n",
|
||||
" \n",
|
||||
" # Simple ASCII plot for loss\n",
|
||||
" train_loss = history['train_loss']\n",
|
||||
" val_loss = history.get('val_loss', [])\n",
|
||||
" \n",
|
||||
" if train_loss:\n",
|
||||
" min_loss = min(train_loss)\n",
|
||||
" max_loss = max(train_loss)\n",
|
||||
" \n",
|
||||
" print(f\"Train Loss: {train_loss[0]:.4f} → {train_loss[-1]:.4f}\")\n",
|
||||
" if val_loss:\n",
|
||||
" print(f\"Val Loss: {val_loss[0]:.4f} → {val_loss[-1]:.4f}\")\n",
|
||||
" \n",
|
||||
" # Show trend\n",
|
||||
" if train_loss[-1] < train_loss[0]:\n",
|
||||
" print(\"✅ Training loss decreased (model is learning!)\")\n",
|
||||
" else:\n",
|
||||
" print(\"⚠️ Training loss increased (check learning rate)\")\n",
|
||||
" \n",
|
||||
" if val_loss and len(val_loss) > 1:\n",
|
||||
" if val_loss[-1] > val_loss[-2]:\n",
|
||||
" print(\"⚠️ Validation loss increasing (possible overfitting)\")\n",
|
||||
" \n",
|
||||
" print(\"-\" * 50)\n",
|
||||
" ### END SOLUTION"
|
||||
"**Ready for compression?** Your training pipelines are now ready for real-world deployment!"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user