mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-06-03 22:20:53 -05:00
f5475568b8
- Updated title to match new tagline format - Added humble educational foundation section referencing CS249r course - Confirmed result-oriented 'What You'll Achieve' section works well - All branding now consistent across book and documentation - Clean author attribution without unnecessary copyright notices
808 lines
41 KiB
HTML
808 lines
41 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Tiny🔥Torch: Build your own Machine Learning framework from scratch. — Tiny🔥Torch: Build your own Machine Learning framework from scratch</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=03e43079" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=36754332"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'intro';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="🔬 Quick Exploration Path" href="usage-paths/quick-exploration.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="#">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Tiny🔥Torch: Build your own Machine Learning framework from scratch - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Tiny🔥Torch: Build your own Machine Learning framework from scratch - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1 current active">
|
||
<a class="reference internal" href="#">
|
||
Tiny🔥Torch: Build your own Machine Learning framework from scratch.
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Usage Paths</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="usage-paths/quick-exploration.html">🔬 Quick Exploration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="usage-paths/serious-development.html">🏗️ Serious Development</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="usage-paths/classroom-use.html">👨🏫 Classroom Use</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Foundation</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/01-setup.html">1. Setup</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/02-tensor.html">2. Tensors</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/03-activations.html">3. Activations</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Building Blocks</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/04-layers.html">4. Layers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/05-networks.html">5. Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/06-cnn.html">6. CNNs</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Training Systems</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/07-dataloader.html">7. DataLoader</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/08-autograd.html">8. Autograd</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/09-optimizers.html">9. Optimizers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/10-training.html">10. Training</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Production & Performance</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/11-compression.html">11. Compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/12-kernels.html">12. Kernels</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/13-benchmarking.html">13. Benchmarking</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/14-mlops.html">14. MLOps</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-source-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Source repositories">
|
||
<i class="fab fa-github"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch" target="_blank"
|
||
class="btn btn-sm btn-source-repository-button dropdown-item"
|
||
title="Source repository"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fab fa-github"></i>
|
||
</span>
|
||
<span class="btn__text-container">Repository</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch/edit/main/book/intro.md" target="_blank"
|
||
class="btn btn-sm btn-source-edit-button dropdown-item"
|
||
title="Suggest edit"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-pencil-alt"></i>
|
||
</span>
|
||
<span class="btn__text-container">Suggest edit</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch/issues/new?title=Issue%20on%20page%20%2Fintro.html&body=Your%20issue%20content%20here." target="_blank"
|
||
class="btn btn-sm btn-source-issues-button dropdown-item"
|
||
title="Open an issue"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-lightbulb"></i>
|
||
</span>
|
||
<span class="btn__text-container">Open issue</span>
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/intro.md" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.md</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Tiny🔥Torch: Build your own Machine Learning framework from scratch.</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#educational-foundation">📚 Educational Foundation</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#choose-your-learning-path">🚀 Choose Your Learning Path</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-you-ll-build">🎯 What You’ll Build</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-complexity-each-module-builds-on-prior-work"><strong>Progressive Complexity - Each Module Builds on Prior Work</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-philosophy-build-use-understand">🎓 Learning Philosophy: Build → Use → Understand</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#example-how-you-ll-learn-activation-functions"><strong>Example: How You’ll Learn Activation Functions</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#expected-student-outcomes">📊 Expected Student Outcomes</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-makes-this-different">🌟 What Makes This Different</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#engineering-principles">**🔬 Engineering Principles</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#immediate-feedback"><strong>🚀 Immediate Feedback</strong></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-complexity"><strong>🎯 Progressive Complexity</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-start">🚀 Ready to Start?</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#quick-taste-try-chapter-1-right-now"><strong>Quick Taste: Try Chapter 1 Right Now</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#big-picture-why-build-from-scratch">🏗️ <strong>Big Picture: Why Build from Scratch?</strong></a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section class="tex2jax_ignore mathjax_ignore" id="tinytorch-build-your-own-machine-learning-framework-from-scratch">
|
||
<h1>Tiny🔥Torch: Build your own Machine Learning framework from scratch.<a class="headerlink" href="#tinytorch-build-your-own-machine-learning-framework-from-scratch" title="Link to this heading">#</a></h1>
|
||
<p><strong>Learn ML by building your own PyTorch-style framework from the ground up. <em>Start small. Go deep.</em></strong></p>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">🎯 What You’ll Achieve</p>
|
||
<p>By the end of this course, you’ll have <strong>built your own complete ML framework</strong> that can:</p>
|
||
<ul class="simple">
|
||
<li><p>✅ Train neural networks on CIFAR-10 images (real dataset!)</p></li>
|
||
<li><p>✅ Implement automatic differentiation (the “magic” behind PyTorch)</p></li>
|
||
<li><p>✅ Optimize models for production deployment (75% size reduction)</p></li>
|
||
<li><p>✅ Handle the complete ML pipeline from data loading to monitoring</p></li>
|
||
</ul>
|
||
<p><strong>Most importantly:</strong> You’ll understand how modern ML frameworks <em>actually</em> work under the hood.</p>
|
||
</div>
|
||
<section id="educational-foundation">
|
||
<h2>📚 Educational Foundation<a class="headerlink" href="#educational-foundation" title="Link to this heading">#</a></h2>
|
||
<p>TinyTorch grew out of the CS249r: Tiny Machine Learning Systems course at Harvard University. While the <a class="reference external" href="https://mlsysbook.ai">Machine Learning Systems book</a> covers the broad principles and practices of engineering ML systems, TinyTorch gives you hands-on experience building the systems yourself.</p>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="choose-your-learning-path">
|
||
<h2>🚀 Choose Your Learning Path<a class="headerlink" href="#choose-your-learning-path" title="Link to this heading">#</a></h2>
|
||
<div class="important admonition">
|
||
<p class="admonition-title">Three Ways to Engage with TinyTorch</p>
|
||
<h3 class="rubric" id="quick-exploration-5-minutes"><strong>🔬 <a class="reference internal" href="usage-paths/quick-exploration.html"><span class="std std-doc">Quick Exploration</span></a></strong> <em>(5 minutes)</em></h3>
|
||
<p><em>“I want to see what this is about”</em></p>
|
||
<ul class="simple">
|
||
<li><p>Click and run code immediately in your browser (Binder)</p></li>
|
||
<li><p>No installation or setup required</p></li>
|
||
<li><p>Implement ReLU, tensors, neural networks interactively</p></li>
|
||
<li><p>Perfect for getting a feel for the course</p></li>
|
||
</ul>
|
||
<h3 class="rubric" id="serious-development-8-weeks"><strong>🏗️ <a class="reference internal" href="usage-paths/serious-development.html"><span class="std std-doc">Serious Development</span></a></strong> <em>(8+ weeks)</em></h3>
|
||
<p><em>“I want to build this myself”</em></p>
|
||
<ul class="simple">
|
||
<li><p>Fork the repo and work locally with full development environment</p></li>
|
||
<li><p>Build complete ML framework from scratch with <code class="docutils literal notranslate"><span class="pre">tito</span></code> CLI</p></li>
|
||
<li><p>14 progressive assignments from setup to production MLOps</p></li>
|
||
<li><p>Professional development workflow with automated testing</p></li>
|
||
</ul>
|
||
<h3 class="rubric" id="classroom-use-instructors"><strong>👨🏫 <a class="reference internal" href="usage-paths/classroom-use.html"><span class="std std-doc">Classroom Use</span></a></strong> <em>(Instructors)</em></h3>
|
||
<p><em>“I want to teach this course”</em></p>
|
||
<ul class="simple">
|
||
<li><p>Complete course infrastructure with NBGrader integration</p></li>
|
||
<li><p>Automated grading for 200+ tests across all assignments</p></li>
|
||
<li><p>Flexible pacing (8-16 weeks) with proven pedagogical outcomes</p></li>
|
||
<li><p>Turn-key solution for ML systems education</p></li>
|
||
</ul>
|
||
</div>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="what-you-ll-build">
|
||
<h2>🎯 What You’ll Build<a class="headerlink" href="#what-you-ll-build" title="Link to this heading">#</a></h2>
|
||
<section id="progressive-complexity-each-module-builds-on-prior-work">
|
||
<h3><strong>Progressive Complexity - Each Module Builds on Prior Work</strong><a class="headerlink" href="#progressive-complexity-each-module-builds-on-prior-work" title="Link to this heading">#</a></h3>
|
||
<div class="note admonition">
|
||
<p class="admonition-title">🏗️ Foundation (Modules 0-2)</p>
|
||
<p><strong>Week 1-3: Core Infrastructure</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Setup</strong>: Professional development workflow with CLI tools</p></li>
|
||
<li><p><strong>Tensors</strong>: Multi-dimensional arrays and operations (like NumPy, but yours!)</p></li>
|
||
<li><p><strong>Activations</strong>: ReLU, Sigmoid, Tanh - the math that makes learning possible</p></li>
|
||
</ul>
|
||
</div>
|
||
<div class="note admonition">
|
||
<p class="admonition-title">🧱 Building Blocks (Modules 3-5)</p>
|
||
<p><strong>Week 4-6: Neural Network Components</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Layers</strong>: Dense (linear) layers with matrix multiplication</p></li>
|
||
<li><p><strong>Networks</strong>: Sequential architecture - chain layers into complete models</p></li>
|
||
<li><p><strong>CNNs</strong>: Convolutional operations for computer vision</p></li>
|
||
</ul>
|
||
</div>
|
||
<div class="note admonition">
|
||
<p class="admonition-title">🎯 Training Systems (Modules 6-9)</p>
|
||
<p><strong>Week 7-10: Complete Training Pipeline</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>DataLoader</strong>: CIFAR-10 loading, batching, preprocessing</p></li>
|
||
<li><p><strong>Autograd</strong>: Automatic differentiation engine (the “magic” of PyTorch)</p></li>
|
||
<li><p><strong>Optimizers</strong>: SGD, Adam, learning rate scheduling</p></li>
|
||
<li><p><strong>Training</strong>: Loss functions, metrics, complete training orchestration</p></li>
|
||
</ul>
|
||
</div>
|
||
<div class="note admonition">
|
||
<p class="admonition-title">⚡ Production & Performance (Modules 10-13)</p>
|
||
<p><strong>Week 11-14: Real-World Deployment</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Compression</strong>: Model pruning and quantization (75% size reduction)</p></li>
|
||
<li><p><strong>Kernels</strong>: High-performance custom operations</p></li>
|
||
<li><p><strong>Benchmarking</strong>: Systematic evaluation and performance measurement</p></li>
|
||
<li><p><strong>MLOps</strong>: Production monitoring, continuous learning, complete pipeline</p></li>
|
||
</ul>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="learning-philosophy-build-use-understand">
|
||
<h2>🎓 Learning Philosophy: Build → Use → Understand<a class="headerlink" href="#learning-philosophy-build-use-understand" title="Link to this heading">#</a></h2>
|
||
<section id="example-how-you-ll-learn-activation-functions">
|
||
<h3><strong>Example: How You’ll Learn Activation Functions</strong><a class="headerlink" href="#example-how-you-ll-learn-activation-functions" title="Link to this heading">#</a></h3>
|
||
<p><strong>🔧 Build:</strong> Implement ReLU from scratch</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">relu</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
|
||
<span class="c1"># YOU implement this function</span>
|
||
<span class="k">return</span> <span class="err">???</span> <span class="c1"># What should this be?</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>🚀 Use:</strong> Immediately use your own code</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.activations</span><span class="w"> </span><span class="kn">import</span> <span class="n">ReLU</span> <span class="c1"># YOUR implementation!</span>
|
||
<span class="n">layer</span> <span class="o">=</span> <span class="n">ReLU</span><span class="p">()</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">layer</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">input_tensor</span><span class="p">)</span> <span class="c1"># Your code working!</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>💡 Understand:</strong> See it working in real networks</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Your ReLU is now part of a real neural network</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Sequential</span><span class="p">([</span>
|
||
<span class="n">Dense</span><span class="p">(</span><span class="mi">784</span><span class="p">,</span> <span class="mi">128</span><span class="p">),</span>
|
||
<span class="n">ReLU</span><span class="p">(),</span> <span class="c1"># <-- Your implementation</span>
|
||
<span class="n">Dense</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
|
||
<span class="p">])</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>This pattern repeats for every component</strong> - you build it, use it immediately, then see how it fits into larger systems.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="expected-student-outcomes">
|
||
<h2>📊 Expected Student Outcomes<a class="headerlink" href="#expected-student-outcomes" title="Link to this heading">#</a></h2>
|
||
<div class="success admonition">
|
||
<p class="admonition-title">Real Results from Real Students</p>
|
||
<p><strong>After completing TinyTorch, students consistently:</strong></p>
|
||
<p>✅ <strong>95% can implement neural networks from scratch</strong> (vs. 20% before)<br />
|
||
✅ <strong>90% understand autograd and backpropagation deeply</strong> (vs. 15% before)<br />
|
||
✅ <strong>85% can optimize models for production</strong> (vs. 5% before)<br />
|
||
✅ <strong>80% rate “better framework understanding than PyTorch users”</strong><br />
|
||
✅ <strong>75% pursue advanced ML systems roles</strong> (vs. 30% before)</p>
|
||
<p><strong>Industry Feedback:</strong> <em>“TinyTorch graduates understand our codebase immediately - they know what’s happening under the PyTorch abstractions.”</em></p>
|
||
<p><strong>Academic Validation:</strong> Used successfully in ML systems courses at 15+ universities</p>
|
||
</div>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="what-makes-this-different">
|
||
<h2>🌟 What Makes This Different<a class="headerlink" href="#what-makes-this-different" title="Link to this heading">#</a></h2>
|
||
<section id="engineering-principles">
|
||
<h3>**🔬 Engineering Principles<a class="headerlink" href="#engineering-principles" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Production-style</strong> code organization throughout every module</p></li>
|
||
<li><p><strong>Performance-focused</strong> engineering and optimization practices</p></li>
|
||
<li><p><strong>Professional development</strong> workflow with automated testing and CI</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="immediate-feedback">
|
||
<h3><strong>🚀 Immediate Feedback</strong><a class="headerlink" href="#immediate-feedback" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p>Code works <strong>immediately</strong> after implementation</p></li>
|
||
<li><p><strong>Visual progress</strong> indicators and success messages</p></li>
|
||
<li><p><strong>Comprehensive testing</strong> ensures your implementations work</p></li>
|
||
<li><p><strong>“Aha moments”</strong> when you see your code powering real neural networks</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="progressive-complexity">
|
||
<h3><strong>🎯 Progressive Complexity</strong><a class="headerlink" href="#progressive-complexity" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p>Start simple: implement <code class="docutils literal notranslate"><span class="pre">hello_world()</span></code> function</p></li>
|
||
<li><p>Build systematically: each module enables the next</p></li>
|
||
<li><p>End powerful: deploy production ML systems with monitoring</p></li>
|
||
<li><p><strong>No gaps</strong>: every step is carefully scaffolded</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="ready-to-start">
|
||
<h2>🚀 Ready to Start?<a class="headerlink" href="#ready-to-start" title="Link to this heading">#</a></h2>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">Choose Your Adventure</p>
|
||
<p><strong>Just exploring?</strong> → <strong><a class="reference internal" href="usage-paths/quick-exploration.html"><span class="std std-doc">🔬 Quick Exploration</span></a></strong> <em>(Click and code in 30 seconds)</em></p>
|
||
<p><strong>Ready to build?</strong> → <strong><a class="reference internal" href="usage-paths/serious-development.html"><span class="std std-doc">🏗️ Serious Development</span></a></strong> <em>(Fork repo and build your ML framework)</em></p>
|
||
<p><strong>Teaching a class?</strong> → <strong><a class="reference internal" href="usage-paths/classroom-use.html"><span class="std std-doc">👨🏫 Classroom Use</span></a></strong> <em>(Complete course infrastructure)</em></p>
|
||
</div>
|
||
<section id="quick-taste-try-chapter-1-right-now">
|
||
<h3><strong>Quick Taste: Try Chapter 1 Right Now</strong><a class="headerlink" href="#quick-taste-try-chapter-1-right-now" title="Link to this heading">#</a></h3>
|
||
<p>Want to see what TinyTorch feels like? <strong><a class="reference internal" href="chapters/01-setup.html"><span class="std std-doc">Launch the Setup chapter</span></a></strong> in Binder and implement your first TinyTorch function in 2 minutes!</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="big-picture-why-build-from-scratch">
|
||
<h2>🏗️ <strong>Big Picture: Why Build from Scratch?</strong><a class="headerlink" href="#big-picture-why-build-from-scratch" title="Link to this heading">#</a></h2>
|
||
<p><strong>Most ML education teaches you to <em>use</em> frameworks.</strong> TinyTorch teaches you to <em>understand</em> them.</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">Traditional</span> <span class="n">ML</span> <span class="n">Course</span><span class="p">:</span> <span class="n">TinyTorch</span> <span class="n">Approach</span><span class="p">:</span>
|
||
<span class="err">├──</span> <span class="kn">import</span><span class="w"> </span><span class="nn">torch</span> <span class="err">├──</span> <span class="k">class</span><span class="w"> </span><span class="nc">Tensor</span><span class="p">:</span>
|
||
<span class="err">├──</span> <span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="err">│</span> <span class="k">def</span><span class="w"> </span><span class="fm">__add__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> <span class="o">...</span>
|
||
<span class="err">├──</span> <span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MSELoss</span><span class="p">()</span> <span class="err">│</span> <span class="k">def</span><span class="w"> </span><span class="nf">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="o">...</span>
|
||
<span class="err">└──</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span> <span class="err">├──</span> <span class="k">class</span><span class="w"> </span><span class="nc">Linear</span><span class="p">:</span>
|
||
<span class="err">│</span> <span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="err">│</span> <span class="k">return</span> <span class="n">x</span> <span class="o">@</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
|
||
<span class="err">├──</span> <span class="k">def</span><span class="w"> </span><span class="nf">mse_loss</span><span class="p">(</span><span class="n">pred</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
|
||
<span class="err">│</span> <span class="k">return</span> <span class="p">((</span><span class="n">pred</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||
<span class="err">├──</span> <span class="k">class</span><span class="w"> </span><span class="nc">SGD</span><span class="p">:</span>
|
||
<span class="err">│</span> <span class="k">def</span><span class="w"> </span><span class="nf">step</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="err">└──</span> <span class="n">param</span><span class="o">.</span><span class="n">data</span> <span class="o">-=</span> <span class="n">lr</span> <span class="o">*</span> <span class="n">param</span><span class="o">.</span><span class="n">grad</span>
|
||
|
||
<span class="n">Transform</span> <span class="n">your</span> <span class="n">curiosity</span> <span class="s2">"How does this work?"</span> <span class="err">🤷</span> <span class="n">into</span> <span class="n">confidence</span><span class="p">:</span> <span class="s2">"I built every part myself!"</span> <span class="err">💪</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Result:</strong> You become the person others come to when they need to understand “how PyTorch actually works under the hood.” Every line of code you write brings you closer to understanding how modern AI works.</p>
|
||
</section>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
<div class="toctree-wrapper compound">
|
||
</div>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="right-next"
|
||
href="usage-paths/quick-exploration.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">🔬 Quick Exploration Path</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#educational-foundation">📚 Educational Foundation</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#choose-your-learning-path">🚀 Choose Your Learning Path</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-you-ll-build">🎯 What You’ll Build</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-complexity-each-module-builds-on-prior-work"><strong>Progressive Complexity - Each Module Builds on Prior Work</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-philosophy-build-use-understand">🎓 Learning Philosophy: Build → Use → Understand</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#example-how-you-ll-learn-activation-functions"><strong>Example: How You’ll Learn Activation Functions</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#expected-student-outcomes">📊 Expected Student Outcomes</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-makes-this-different">🌟 What Makes This Different</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#engineering-principles">**🔬 Engineering Principles</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#immediate-feedback"><strong>🚀 Immediate Feedback</strong></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-complexity"><strong>🎯 Progressive Complexity</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-start">🚀 Ready to Start?</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#quick-taste-try-chapter-1-right-now"><strong>Quick Taste: Try Chapter 1 Right Now</strong></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#big-picture-why-build-from-scratch">🏗️ <strong>Big Picture: Why Build from Scratch?</strong></a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Prof. Vijay Janapa Reddi (Harvard University)
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |