mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-05-23 19:35:56 -05:00
1006 lines
57 KiB
HTML
1006 lines
57 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Frequently Asked Questions — Tiny🔥Torch</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=03e43079" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
<link rel="stylesheet" type="text/css" href="_static/custom.css?v=009d37f4" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'faq';</script>
|
||
<script src="_static/ml-timeline.js?v=76e9b3e3"></script>
|
||
<script src="_static/wip-banner.js?v=04a7e74d"></script>
|
||
<script src="_static/marimo-badges.js?v=e6289128"></script>
|
||
<script src="_static/sidebar-link.js?v=404b701b"></script>
|
||
<script src="_static/hero-carousel.js?v=10341d2a"></script>
|
||
<script src="_static/subscribe-modal.js?v=42919b64"></script>
|
||
<link rel="icon" href="_static/favicon.svg"/>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="TITO Command Reference" href="tito/overview.html" />
|
||
<link rel="prev" title="Journey Through ML History" href="chapters/milestones.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search..."
|
||
aria-label="Search..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo-tinytorch.png" class="logo__image only-light" alt="Tiny🔥Torch - Home"/>
|
||
<script>document.write(`<img src="_static/logo-tinytorch.png" class="logo__image only-dark" alt="Tiny🔥Torch - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🚀 Getting Started</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="getting-started.html">Complete Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏗 Foundation Tier (01-07)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="tiers/foundation.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/01_tensor_ABOUT.html">01. Tensor</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/02_activations_ABOUT.html">02. Activations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/03_layers_ABOUT.html">03. Layers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/04_losses_ABOUT.html">04. Losses</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/05_autograd_ABOUT.html">05. Autograd</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/06_optimizers_ABOUT.html">06. Optimizers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/07_training_ABOUT.html">07. Training</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏛️ Architecture Tier (08-13)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="tiers/architecture.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/08_dataloader_ABOUT.html">08. DataLoader</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/09_spatial_ABOUT.html">09. Convolutions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/10_tokenization_ABOUT.html">10. Tokenization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/11_embeddings_ABOUT.html">11. Embeddings</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/12_attention_ABOUT.html">12. Attention</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/13_transformers_ABOUT.html">13. Transformers</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">⏱️ Optimization Tier (14-19)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="tiers/optimization.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/14_profiling_ABOUT.html">14. Profiling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/15_quantization_ABOUT.html">15. Quantization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/16_compression_ABOUT.html">16. Compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/17_memoization_ABOUT.html">17. Memoization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/18_acceleration_ABOUT.html">18. Acceleration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/19_benchmarking_ABOUT.html">19. Benchmarking</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏅 Capstone Competition</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="tiers/olympics.html">📖 Competition Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="modules/20_capstone_ABOUT.html">20. Torch Olympics</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🧭 Course Orientation</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/00-introduction.html">Course Structure</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="prerequisites.html">Prerequisites & Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/learning-journey.html">Learning Journey</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapters/milestones.html">Historical Milestones</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">FAQ</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🛠️ TITO CLI Reference</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="tito/overview.html">Command Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="tito/modules.html">Module Workflow</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="tito/milestones.html">Milestone System</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="tito/data.html">Progress & Data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="tito/troubleshooting.html">Troubleshooting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="datasets.html">Datasets Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🤝 Community</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="community.html">Ecosystem</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="resources.html">Learning Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="credits.html">Credits & Acknowledgments</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/faq.md" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.md</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Frequently Asked Questions</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#general-questions">General Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-is-tinytorch">What is TinyTorch?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#who-is-tinytorch-for">Who is TinyTorch for?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-long-does-it-take">How long does it take?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-tinytorch-vs-alternatives">Why TinyTorch vs. Alternatives?</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-not-just-use-pytorch-or-tensorflow-directly">Why not just use PyTorch or TensorFlow directly?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-tinytorch-instead-of-andrej-karpathys-micrograd-or-nanogpt">Why TinyTorch instead of Andrej Karpathy’s micrograd or nanoGPT?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-not-just-read-pytorch-source-code">Why not just read PyTorch source code?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#technical-questions">Technical Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-programming-background-do-i-need">What programming background do I need?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-hardware-do-i-need">What hardware do I need?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#does-tinytorch-replace-a-traditional-ml-course">Does TinyTorch replace a traditional ML course?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-use-tinytorch-for-research-or-production">Can I use TinyTorch for research or production?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#course-structure-questions">Course Structure Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#do-i-need-to-complete-all-20-modules">Do I need to complete all 20 modules?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-are-the-milestones">What are the milestones?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#are-the-checkpoints-required">Are the checkpoints required?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#practical-questions">Practical Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-do-i-get-started">How do I get started?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#whats-the-typical-workflow">What’s the typical workflow?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-use-this-in-my-classroom">Can I use this in my classroom?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-do-i-get-help">How do I get help?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#philosophy-questions">Philosophy Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-build-from-scratch-instead-of-using-libraries">Why build from scratch instead of using libraries?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#isnt-this-reinventing-the-wheel">Isn’t this reinventing the wheel?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#will-i-still-use-pytorch-tensorflow-after-this">Will I still use PyTorch/TensorFlow after this?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#community-questions">Community Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-contribute-to-tinytorch">Can I contribute to TinyTorch?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#is-there-a-community">Is there a community?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-is-tinytorch-maintained">How is TinyTorch maintained?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#still-have-questions">Still Have Questions?</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section id="frequently-asked-questions">
|
||
<h1>Frequently Asked Questions<a class="headerlink" href="#frequently-asked-questions" title="Link to this heading">#</a></h1>
|
||
<div style="background: #f8f9fa; padding: 2rem; border-radius: 0.5rem; margin: 2rem 0; text-align: center;">
|
||
<h2 style="margin: 0 0 1rem 0; color: #495057;">Common Questions About TinyTorch</h2>
|
||
<p style="margin: 0; font-size: 1.1rem; color: #6c757d;">Why build from scratch? Why not just use PyTorch? All your questions answered.</p>
|
||
</div>
|
||
<section id="general-questions">
|
||
<h2>General Questions<a class="headerlink" href="#general-questions" title="Link to this heading">#</a></h2>
|
||
<section id="what-is-tinytorch">
|
||
<h3>What is TinyTorch?<a class="headerlink" href="#what-is-tinytorch" title="Link to this heading">#</a></h3>
|
||
<p>TinyTorch is an educational ML systems framework where you build a complete neural network library from scratch. Instead of using PyTorch or TensorFlow as black boxes, you implement every component yourself—tensors, gradients, optimizers, attention mechanisms—gaining deep understanding of how modern ML frameworks actually work.</p>
|
||
</section>
|
||
<section id="who-is-tinytorch-for">
|
||
<h3>Who is TinyTorch for?<a class="headerlink" href="#who-is-tinytorch-for" title="Link to this heading">#</a></h3>
|
||
<p>TinyTorch is designed for:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Students</strong> learning ML who want to understand what’s happening under the hood</p></li>
|
||
<li><p><strong>ML practitioners</strong> who want to debug models more effectively</p></li>
|
||
<li><p><strong>Systems engineers</strong> building or optimizing ML infrastructure</p></li>
|
||
<li><p><strong>Researchers</strong> who need to implement novel architectures</p></li>
|
||
<li><p><strong>Educators</strong> teaching ML systems (not just ML algorithms)</p></li>
|
||
</ul>
|
||
<p>If you’ve ever wondered “why does my model OOM?” or “how does autograd actually work?”, TinyTorch is for you.</p>
|
||
</section>
|
||
<section id="how-long-does-it-take">
|
||
<h3>How long does it take?<a class="headerlink" href="#how-long-does-it-take" title="Link to this heading">#</a></h3>
|
||
<p><strong>Quick exploration</strong>: 2-4 weeks focusing on Foundation Tier (Modules 01-07)
|
||
<strong>Complete course</strong>: 14-18 weeks implementing all three tiers (20 modules)
|
||
<strong>Flexible approach</strong>: Pick specific modules based on your learning goals</p>
|
||
<p>You control the pace. Some students complete it in intensive 8-week sprints, others spread it across a semester.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="why-tinytorch-vs-alternatives">
|
||
<h2>Why TinyTorch vs. Alternatives?<a class="headerlink" href="#why-tinytorch-vs-alternatives" title="Link to this heading">#</a></h2>
|
||
<section id="why-not-just-use-pytorch-or-tensorflow-directly">
|
||
<h3>Why not just use PyTorch or TensorFlow directly?<a class="headerlink" href="#why-not-just-use-pytorch-or-tensorflow-directly" title="Link to this heading">#</a></h3>
|
||
<p><strong>Short answer</strong>: Because using a library doesn’t teach you how it works.</p>
|
||
<p><strong>The problem with “just use PyTorch”:</strong></p>
|
||
<p>When you write:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">torch.nn</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">nn</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">784</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
|
||
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">())</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>You’re calling functions you don’t understand. When things break (and they will), you’re stuck:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>OOM errors</strong>: Why? How much memory does this need?</p></li>
|
||
<li><p><strong>Slow training</strong>: What’s the bottleneck? Data loading? Computation?</p></li>
|
||
<li><p><strong>NaN losses</strong>: Where did gradients explode? How do you debug?</p></li>
|
||
</ul>
|
||
<p><strong>What TinyTorch teaches:</strong></p>
|
||
<p>When you implement <code class="docutils literal notranslate"><span class="pre">Linear</span></code> yourself:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">Linear</span><span class="p">:</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">):</span>
|
||
<span class="c1"># You understand EXACTLY what memory is allocated</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">randn</span><span class="p">(</span><span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">)</span> <span class="o">*</span> <span class="mf">0.01</span> <span class="c1"># Why 0.01?</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">zeros</span><span class="p">(</span><span class="n">out_features</span><span class="p">)</span> <span class="c1"># Why zeros?</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">input</span> <span class="o">=</span> <span class="n">x</span> <span class="c1"># Why save input? (Hint: backward pass)</span>
|
||
<span class="k">return</span> <span class="n">x</span> <span class="o">@</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="c1"># You know the exact operations</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">grad</span><span class="p">):</span>
|
||
<span class="c1"># You wrote this gradient! You can debug it!</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">input</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">grad</span>
|
||
<span class="k">return</span> <span class="n">grad</span> <span class="o">@</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">T</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Now you can:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Calculate memory requirements</strong> before running</p></li>
|
||
<li><p><strong>Profile and optimize</strong> every operation</p></li>
|
||
<li><p><strong>Debug gradient issues</strong> by inspecting your own code</p></li>
|
||
<li><p><strong>Implement novel architectures</strong> with confidence</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="why-tinytorch-instead-of-andrej-karpathys-micrograd-or-nanogpt">
|
||
<h3>Why TinyTorch instead of Andrej Karpathy’s micrograd or nanoGPT?<a class="headerlink" href="#why-tinytorch-instead-of-andrej-karpathys-micrograd-or-nanogpt" title="Link to this heading">#</a></h3>
|
||
<p>We love micrograd and nanoGPT! They’re excellent educational resources. Here’s how TinyTorch differs:</p>
|
||
<p><strong>micrograd (100 lines)</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Scope</strong>: Teaches autograd elegantly in minimal code</p></li>
|
||
<li><p><strong>Limitation</strong>: Doesn’t cover CNNs, transformers, data loading, optimization</p></li>
|
||
<li><p><strong>Use case</strong>: Perfect introduction to automatic differentiation</p></li>
|
||
</ul>
|
||
<p><strong>nanoGPT (300 lines)</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Scope</strong>: Clean GPT implementation for understanding transformers</p></li>
|
||
<li><p><strong>Limitation</strong>: Doesn’t teach fundamentals (tensors, layers, training loops)</p></li>
|
||
<li><p><strong>Use case</strong>: Excellent for understanding transformer architecture specifically</p></li>
|
||
</ul>
|
||
<p><strong>TinyTorch (20 modules, complete framework)</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Scope</strong>: Full ML systems course from mathematical primitives to production deployment</p></li>
|
||
<li><p><strong>Coverage</strong>:</p>
|
||
<ul>
|
||
<li><p>Foundation (tensors, autograd, optimizers)</p></li>
|
||
<li><p>Architecture (CNNs for vision, transformers for language)</p></li>
|
||
<li><p>Optimization (profiling, quantization, benchmarking)</p></li>
|
||
</ul>
|
||
</li>
|
||
<li><p><strong>Outcome</strong>: You build a unified framework supporting both vision AND language models</p></li>
|
||
<li><p><strong>Systems focus</strong>: Memory profiling, performance analysis, and production context built into every module</p></li>
|
||
</ul>
|
||
<p><strong>Analogy:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>micrograd</strong>: Learn how an engine works</p></li>
|
||
<li><p><strong>nanoGPT</strong>: Learn how a sports car works</p></li>
|
||
<li><p><strong>TinyTorch</strong>: Build a complete vehicle manufacturing plant (and understand engines, cars, AND the factory)</p></li>
|
||
</ul>
|
||
<p><strong>When to use each:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Start with micrograd</strong> if you want a gentle introduction to autograd (1-2 hours)</p></li>
|
||
<li><p><strong>Try nanoGPT</strong> if you specifically want to understand GPT architecture (1-2 days)</p></li>
|
||
<li><p><strong>Choose TinyTorch</strong> if you want complete ML systems engineering skills (8-18 weeks)</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="why-not-just-read-pytorch-source-code">
|
||
<h3>Why not just read PyTorch source code?<a class="headerlink" href="#why-not-just-read-pytorch-source-code" title="Link to this heading">#</a></h3>
|
||
<p><strong>Three problems with reading production framework code:</strong></p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Complexity</strong>: PyTorch has 350K+ lines optimized for production, not learning</p></li>
|
||
<li><p><strong>C++/CUDA</strong>: Core operations are in low-level languages for performance</p></li>
|
||
<li><p><strong>No learning path</strong>: Where do you even start?</p></li>
|
||
</ol>
|
||
<p><strong>TinyTorch’s pedagogical approach:</strong></p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Incremental complexity</strong>: Start with 2D matrices, build up to 4D tensors</p></li>
|
||
<li><p><strong>Pure Python</strong>: Understand algorithms before optimization</p></li>
|
||
<li><p><strong>Guided curriculum</strong>: Clear progression from basics to advanced</p></li>
|
||
<li><p><strong>Systems thinking</strong>: Every module includes profiling and performance analysis</p></li>
|
||
</ol>
|
||
<p>You learn the <em>concepts</em> in TinyTorch, then understand how PyTorch optimizes them for production.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="technical-questions">
|
||
<h2>Technical Questions<a class="headerlink" href="#technical-questions" title="Link to this heading">#</a></h2>
|
||
<section id="what-programming-background-do-i-need">
|
||
<h3>What programming background do I need?<a class="headerlink" href="#what-programming-background-do-i-need" title="Link to this heading">#</a></h3>
|
||
<p><strong>Required:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Python programming (functions, classes, basic NumPy)</p></li>
|
||
<li><p>Basic calculus (derivatives, chain rule)</p></li>
|
||
<li><p>Linear algebra (matrix multiplication)</p></li>
|
||
</ul>
|
||
<p><strong>Helpful but not required:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Git version control</p></li>
|
||
<li><p>Command-line comfort</p></li>
|
||
<li><p>Previous ML course (though TinyTorch teaches from scratch)</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="what-hardware-do-i-need">
|
||
<h3>What hardware do I need?<a class="headerlink" href="#what-hardware-do-i-need" title="Link to this heading">#</a></h3>
|
||
<p><strong>Minimum:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Any laptop with 8GB RAM</p></li>
|
||
<li><p>Works on M1/M2 Macs, Intel, AMD</p></li>
|
||
</ul>
|
||
<p><strong>No GPU required!</strong> TinyTorch runs on CPU and teaches concepts that transfer to GPU optimization.</p>
|
||
</section>
|
||
<section id="does-tinytorch-replace-a-traditional-ml-course">
|
||
<h3>Does TinyTorch replace a traditional ML course?<a class="headerlink" href="#does-tinytorch-replace-a-traditional-ml-course" title="Link to this heading">#</a></h3>
|
||
<p><strong>No, it complements it.</strong></p>
|
||
<p><strong>Traditional ML course teaches:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Algorithms (gradient descent, backpropagation)</p></li>
|
||
<li><p>Theory (loss functions, regularization)</p></li>
|
||
<li><p>Applications (classification, generation)</p></li>
|
||
</ul>
|
||
<p><strong>TinyTorch teaches:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Systems (how frameworks work)</p></li>
|
||
<li><p>Implementation (building from scratch)</p></li>
|
||
<li><p>Production (profiling, optimization, deployment)</p></li>
|
||
</ul>
|
||
<p><strong>Best approach</strong>: Take a traditional ML course for theory, use TinyTorch to deeply understand implementation.</p>
|
||
</section>
|
||
<section id="can-i-use-tinytorch-for-research-or-production">
|
||
<h3>Can I use TinyTorch for research or production?<a class="headerlink" href="#can-i-use-tinytorch-for-research-or-production" title="Link to this heading">#</a></h3>
|
||
<p><strong>Research</strong>: Absolutely! Build novel architectures with full control
|
||
<strong>Production</strong>: TinyTorch is educational—use PyTorch/TensorFlow for production scale</p>
|
||
<p><strong>However:</strong> Understanding TinyTorch makes you much better at using production frameworks. You’ll:</p>
|
||
<ul class="simple">
|
||
<li><p>Write more efficient PyTorch code</p></li>
|
||
<li><p>Debug issues faster</p></li>
|
||
<li><p>Understand performance characteristics</p></li>
|
||
<li><p>Make better architectural decisions</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="course-structure-questions">
|
||
<h2>Course Structure Questions<a class="headerlink" href="#course-structure-questions" title="Link to this heading">#</a></h2>
|
||
<section id="do-i-need-to-complete-all-20-modules">
|
||
<h3>Do I need to complete all 20 modules?<a class="headerlink" href="#do-i-need-to-complete-all-20-modules" title="Link to this heading">#</a></h3>
|
||
<p><strong>No!</strong> TinyTorch offers flexible learning paths:</p>
|
||
<p><strong>Three tiers:</strong></p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Foundation (01-07)</strong>: Core ML infrastructure—understand how training works</p></li>
|
||
<li><p><strong>Architecture (08-13)</strong>: Modern AI architectures—CNNs and transformers</p></li>
|
||
<li><p><strong>Optimization (14-20)</strong>: Production deployment—profiling and acceleration</p></li>
|
||
</ol>
|
||
<p><strong>Suggested paths:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>ML student</strong>: Foundation tier gives you deep understanding</p></li>
|
||
<li><p><strong>Systems engineer</strong>: All three tiers teach complete ML systems</p></li>
|
||
<li><p><strong>Researcher</strong>: Focus on Foundation + Architecture for implementation skills</p></li>
|
||
<li><p><strong>Curious learner</strong>: Pick modules that interest you</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="what-are-the-milestones">
|
||
<h3>What are the milestones?<a class="headerlink" href="#what-are-the-milestones" title="Link to this heading">#</a></h3>
|
||
<p>Milestones are historical ML achievements you recreate with YOUR implementations:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>M01: 1957 Perceptron</strong> - First trainable neural network</p></li>
|
||
<li><p><strong>M02: 1969 XOR</strong> - Multi-layer networks solve XOR problem</p></li>
|
||
<li><p><strong>M03: 1986 MLP</strong> - Backpropagation achieves 95%+ on MNIST</p></li>
|
||
<li><p><strong>M04: 1998 CNN</strong> - LeNet-style CNN gets 75%+ on CIFAR-10</p></li>
|
||
<li><p><strong>M05: 2017 Transformer</strong> - GPT-style text generation</p></li>
|
||
<li><p><strong>M06: 2018 Torch Olympics</strong> - Production optimization benchmarking</p></li>
|
||
</ul>
|
||
<p>Each milestone proves your framework works by running actual ML experiments.</p>
|
||
<p><strong>📖 See <a class="reference internal" href="chapters/milestones.html"><span class="std std-doc">Journey Through ML History</span></a></strong> for details.</p>
|
||
</section>
|
||
<section id="are-the-checkpoints-required">
|
||
<h3>Are the checkpoints required?<a class="headerlink" href="#are-the-checkpoints-required" title="Link to this heading">#</a></h3>
|
||
<p><strong>No, they’re optional.</strong></p>
|
||
<p><strong>The essential workflow:</strong></p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>1. Edit modules → 2. Export → 3. Validate with milestones
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Optional checkpoint system:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Tracks 21 capability checkpoints</p></li>
|
||
<li><p>Helpful for self-assessment</p></li>
|
||
<li><p>Use <code class="docutils literal notranslate"><span class="pre">tito</span> <span class="pre">checkpoint</span> <span class="pre">status</span></code> to view progress</p></li>
|
||
</ul>
|
||
<p><strong>📖 See <a class="reference internal" href="tito/modules.html"><span class="std std-doc">Module Workflow</span></a></strong> for the core development cycle.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="practical-questions">
|
||
<h2>Practical Questions<a class="headerlink" href="#practical-questions" title="Link to this heading">#</a></h2>
|
||
<section id="how-do-i-get-started">
|
||
<h3>How do I get started?<a class="headerlink" href="#how-do-i-get-started" title="Link to this heading">#</a></h3>
|
||
<p><strong>Quick start (15 minutes):</strong></p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># 1. Clone repository</span>
|
||
git<span class="w"> </span>clone<span class="w"> </span>https://github.com/mlsysbook/TinyTorch.git
|
||
<span class="nb">cd</span><span class="w"> </span>TinyTorch
|
||
|
||
<span class="c1"># 2. Automated setup</span>
|
||
./setup-environment.sh
|
||
<span class="nb">source</span><span class="w"> </span>activate.sh
|
||
|
||
<span class="c1"># 3. Verify setup</span>
|
||
tito<span class="w"> </span>system<span class="w"> </span>health
|
||
|
||
<span class="c1"># 4. Start first module</span>
|
||
<span class="nb">cd</span><span class="w"> </span>modules/01_tensor
|
||
jupyter<span class="w"> </span>lab<span class="w"> </span>tensor_dev.py
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>📖 See <a class="reference internal" href="getting-started.html"><span class="std std-doc">Getting Started Guide</span></a></strong> for detailed setup.</p>
|
||
</section>
|
||
<section id="whats-the-typical-workflow">
|
||
<h3>What’s the typical workflow?<a class="headerlink" href="#whats-the-typical-workflow" title="Link to this heading">#</a></h3>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># 1. Work on module source</span>
|
||
<span class="nb">cd</span><span class="w"> </span>modules/03_layers
|
||
jupyter<span class="w"> </span>lab<span class="w"> </span>layers_dev.py
|
||
|
||
<span class="c1"># 2. Export when ready</span>
|
||
tito<span class="w"> </span>module<span class="w"> </span><span class="nb">complete</span><span class="w"> </span><span class="m">03</span>
|
||
|
||
<span class="c1"># 3. Validate by running milestones</span>
|
||
<span class="nb">cd</span><span class="w"> </span>../../milestones/01_1957_perceptron
|
||
python<span class="w"> </span>rosenblatt_forward.py<span class="w"> </span><span class="c1"># Uses YOUR implementation!</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>📖 See <a class="reference internal" href="tito/modules.html"><span class="std std-doc">Module Workflow</span></a></strong> for complete details.</p>
|
||
</section>
|
||
<section id="can-i-use-this-in-my-classroom">
|
||
<h3>Can I use this in my classroom?<a class="headerlink" href="#can-i-use-this-in-my-classroom" title="Link to this heading">#</a></h3>
|
||
<p><strong>Yes!</strong> TinyTorch is designed for classroom use.</p>
|
||
<p><strong>Current status:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Students can work through modules individually</p></li>
|
||
<li><p><a class="reference external" href="https://nbgrader.readthedocs.io/">NBGrader</a> integration coming soon for automated grading</p></li>
|
||
<li><p>Instructor tooling under development</p></li>
|
||
</ul>
|
||
<p><strong>📖 See <span class="xref myst">Classroom Use Guide</span></strong> for details.</p>
|
||
</section>
|
||
<section id="how-do-i-get-help">
|
||
<h3>How do I get help?<a class="headerlink" href="#how-do-i-get-help" title="Link to this heading">#</a></h3>
|
||
<p><strong>Resources:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Documentation</strong>: Comprehensive guides for every module</p></li>
|
||
<li><p><strong>GitHub Issues</strong>: Report bugs or ask questions</p></li>
|
||
<li><p><strong>Community</strong>: (Coming soon) Discord/forum for peer support</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="philosophy-questions">
|
||
<h2>Philosophy Questions<a class="headerlink" href="#philosophy-questions" title="Link to this heading">#</a></h2>
|
||
<section id="why-build-from-scratch-instead-of-using-libraries">
|
||
<h3>Why build from scratch instead of using libraries?<a class="headerlink" href="#why-build-from-scratch-instead-of-using-libraries" title="Link to this heading">#</a></h3>
|
||
<p><strong>The difference between using and understanding:</strong></p>
|
||
<p>When you import a library, you’re limited by what it provides. When you build from scratch, you understand the foundations and can create anything.</p>
|
||
<p><strong>Real-world impact:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Debugging</strong>: “My model won’t train” → You know exactly where to look</p></li>
|
||
<li><p><strong>Optimization</strong>: “Training is slow” → You can profile and fix bottlenecks</p></li>
|
||
<li><p><strong>Innovation</strong>: “I need a novel architecture” → You build it confidently</p></li>
|
||
<li><p><strong>Career</strong>: ML systems engineers who understand internals are highly valued</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="isnt-this-reinventing-the-wheel">
|
||
<h3>Isn’t this reinventing the wheel?<a class="headerlink" href="#isnt-this-reinventing-the-wheel" title="Link to this heading">#</a></h3>
|
||
<p><strong>Yes, intentionally!</strong></p>
|
||
<p><strong>The best way to learn engineering:</strong> Build it yourself.</p>
|
||
<ul class="simple">
|
||
<li><p>Car mechanics learn by taking apart engines</p></li>
|
||
<li><p>Civil engineers build bridge models</p></li>
|
||
<li><p>Software engineers implement data structures from scratch</p></li>
|
||
</ul>
|
||
<p><strong>Then</strong> they use production tools with deep understanding.</p>
|
||
</section>
|
||
<section id="will-i-still-use-pytorch-tensorflow-after-this">
|
||
<h3>Will I still use PyTorch/TensorFlow after this?<a class="headerlink" href="#will-i-still-use-pytorch-tensorflow-after-this" title="Link to this heading">#</a></h3>
|
||
<p><strong>Absolutely!</strong> TinyTorch makes you <em>better</em> at using production frameworks.</p>
|
||
<p><strong>Before TinyTorch:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">784</span><span class="p">,</span> <span class="mi">128</span><span class="p">),</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
<span class="c1"># It works but... why 128? What's the memory usage? How does ReLU affect gradients?</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>After TinyTorch:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">784</span><span class="p">,</span> <span class="mi">128</span><span class="p">),</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
<span class="c1"># I know: 784*128 + 128*10 params = ~100K params * 4 bytes = ~400KB</span>
|
||
<span class="c1"># I understand: ReLU zeros negative gradients, affects backprop</span>
|
||
<span class="c1"># I can optimize: Maybe use smaller hidden layer or quantize to INT8</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>You use the same tools, but with systems-level understanding.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="community-questions">
|
||
<h2>Community Questions<a class="headerlink" href="#community-questions" title="Link to this heading">#</a></h2>
|
||
<section id="can-i-contribute-to-tinytorch">
|
||
<h3>Can I contribute to TinyTorch?<a class="headerlink" href="#can-i-contribute-to-tinytorch" title="Link to this heading">#</a></h3>
|
||
<p><strong>Yes!</strong> TinyTorch is open-source and welcomes contributions:</p>
|
||
<ul class="simple">
|
||
<li><p>Bug fixes and improvements</p></li>
|
||
<li><p>Documentation enhancements</p></li>
|
||
<li><p>Additional modules or extensions</p></li>
|
||
<li><p>Educational resources</p></li>
|
||
</ul>
|
||
<p>Check the GitHub repository for contribution guidelines.</p>
|
||
</section>
|
||
<section id="is-there-a-community">
|
||
<h3>Is there a community?<a class="headerlink" href="#is-there-a-community" title="Link to this heading">#</a></h3>
|
||
<p><strong>Growing!</strong> TinyTorch is launching to the community in December 2024.</p>
|
||
<ul class="simple">
|
||
<li><p>GitHub Discussions for Q&A</p></li>
|
||
<li><p>Optional leaderboard for module 20 competition</p></li>
|
||
<li><p>Community showcase (coming soon)</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="how-is-tinytorch-maintained">
|
||
<h3>How is TinyTorch maintained?<a class="headerlink" href="#how-is-tinytorch-maintained" title="Link to this heading">#</a></h3>
|
||
<p>TinyTorch is developed at the intersection of academia and education:</p>
|
||
<ul class="simple">
|
||
<li><p>Research-backed pedagogy</p></li>
|
||
<li><p>Active development and testing</p></li>
|
||
<li><p>Community feedback integration</p></li>
|
||
<li><p>Regular updates and improvements</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<section id="still-have-questions">
|
||
<h2>Still Have Questions?<a class="headerlink" href="#still-have-questions" title="Link to this heading">#</a></h2>
|
||
<div style="background: #f8f9fa; padding: 2rem; border-radius: 0.5rem; margin: 2rem 0; text-align: center;">
|
||
<h3 style="margin: 0 0 1rem 0; color: #495057;">Ready to Start Building?</h3>
|
||
<p style="margin: 0 0 1.5rem 0; color: #6c757d;">Jump in and start implementing ML systems from scratch</p>
|
||
<a href="getting-started.html" style="display: inline-block; background: #007bff; color: white; padding: 0.75rem 1.5rem; border-radius: 0.25rem; text-decoration: none; font-weight: 500; margin-right: 1rem;">Getting Started →</a>
|
||
<a href="intro.html" style="display: inline-block; background: #28a745; color: white; padding: 0.75rem 1.5rem; border-radius: 0.25rem; text-decoration: none; font-weight: 500;">Learn More →</a>
|
||
</div>
|
||
<p><strong>Can’t find your question?</strong> Open an issue on <a class="reference external" href="https://github.com/mlsysbook/TinyTorch/issues">GitHub</a> and we’ll help!</p>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="chapters/milestones.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Journey Through ML History</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="tito/overview.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">TITO Command Reference</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#general-questions">General Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-is-tinytorch">What is TinyTorch?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#who-is-tinytorch-for">Who is TinyTorch for?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-long-does-it-take">How long does it take?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-tinytorch-vs-alternatives">Why TinyTorch vs. Alternatives?</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-not-just-use-pytorch-or-tensorflow-directly">Why not just use PyTorch or TensorFlow directly?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-tinytorch-instead-of-andrej-karpathys-micrograd-or-nanogpt">Why TinyTorch instead of Andrej Karpathy’s micrograd or nanoGPT?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-not-just-read-pytorch-source-code">Why not just read PyTorch source code?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#technical-questions">Technical Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-programming-background-do-i-need">What programming background do I need?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-hardware-do-i-need">What hardware do I need?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#does-tinytorch-replace-a-traditional-ml-course">Does TinyTorch replace a traditional ML course?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-use-tinytorch-for-research-or-production">Can I use TinyTorch for research or production?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#course-structure-questions">Course Structure Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#do-i-need-to-complete-all-20-modules">Do I need to complete all 20 modules?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-are-the-milestones">What are the milestones?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#are-the-checkpoints-required">Are the checkpoints required?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#practical-questions">Practical Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-do-i-get-started">How do I get started?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#whats-the-typical-workflow">What’s the typical workflow?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-use-this-in-my-classroom">Can I use this in my classroom?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-do-i-get-help">How do I get help?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#philosophy-questions">Philosophy Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-build-from-scratch-instead-of-using-libraries">Why build from scratch instead of using libraries?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#isnt-this-reinventing-the-wheel">Isn’t this reinventing the wheel?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#will-i-still-use-pytorch-tensorflow-after-this">Will I still use PyTorch/TensorFlow after this?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#community-questions">Community Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#can-i-contribute-to-tinytorch">Can I contribute to TinyTorch?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#is-there-a-community">Is there a community?</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-is-tinytorch-maintained">How is TinyTorch maintained?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#still-have-questions">Still Have Questions?</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Prof. Vijay Janapa Reddi (Harvard University)
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2025.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |