github-starred/TinyTorch
2
0
Fork 0
mirror of https://github.com/MLSysBook/TinyTorch.git synced 2026-04-28 11:55:04 -05:00
Code Issues 7 Packages Projects Releases Wiki Activity
Files
gh-pages
TinyTorch/chapters/milestones.html
github-actions[bot] 3c159d9fd8 Deploy live site from main branch - 7bc4f6f835 7bc4f6f835
2025-12-05 00:52:38 +00:00

1442 lines
64 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" data-content_root="../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Journey Through ML History &#8212; Tiny🔥Torch</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
<link href="../_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
<link href="../_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" />
<link rel="stylesheet" type="text/css" href="../_static/styles/sphinx-book-theme.css?v=eba8b062" />
<link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=13237357" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css" />
<link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css?v=4fa983c6" />
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
<link rel="stylesheet" type="text/css" href="../_static/custom.css?v=009d37f4" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
<script src="../_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
<script src="../_static/documentation_options.js?v=9eb32ce0"></script>
<script src="../_static/doctools.js?v=9a2dae69"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=f281be69"></script>
<script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
<script>let toggleHintShow = 'Click to show';</script>
<script>let toggleHintHide = 'Click to hide';</script>
<script>let toggleOpenOnPrint = 'true';</script>
<script src="../_static/togglebutton.js?v=4a39c7ea"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
<script src="../_static/design-tabs.js?v=f930bc37"></script>
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
<script async="async" src="../_static/sphinx-thebe.js?v=c100c467"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
<script type="module" src="https://cdn.jsdelivr.net/npm/mermaid@10.6.1/dist/mermaid.esm.min.mjs"></script>
<script type="module" src="https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.2.0/dist/mermaid-layout-elk.esm.min.mjs"></script>
<script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@10.6.1/dist/mermaid.esm.min.mjs";import elkLayouts from "https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.2.0/dist/mermaid-layout-elk.esm.min.mjs";mermaid.registerLayoutLoaders(elkLayouts);mermaid.initialize({startOnLoad:false});</script>
<script src="https://cdn.jsdelivr.net/npm/d3@7.9.0/dist/d3.min.js"></script>
<script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@10.6.1/dist/mermaid.esm.min.mjs";
const defaultStyle = document.createElement('style');
defaultStyle.textContent = `pre.mermaid {
/* Same as .mermaid-container > pre */
display: block;
width: 100%;
}
pre.mermaid > svg {
/* Same as .mermaid-container > pre > svg */
height: 500px;
width: 100%;
max-width: 100% !important;
}
`;
document.head.appendChild(defaultStyle);
const fullscreenStyle = document.createElement('style');
fullscreenStyle.textContent = `.mermaid-container {
display: flex;
flex-direction: row;
width: 100%;
}
.mermaid-container > pre {
display: block;
width: 100%;
}
.mermaid-container > pre > svg {
height: 500px;
width: 100%;
max-width: 100% !important;
}
.mermaid-fullscreen-btn {
width: 28px;
height: 28px;
background: rgba(255, 255, 255, 0.95);
border: 1px solid rgba(0, 0, 0, 0.3);
border-radius: 4px;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: all 0.2s;
box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2);
font-size: 14px;
line-height: 1;
padding: 0;
color: #333;
}
.mermaid-fullscreen-btn:hover {
opacity: 100% !important;
background: rgba(255, 255, 255, 1);
box-shadow: 0 3px 10px rgba(0, 0, 0, 0.3);
transform: scale(1.1);
}
.mermaid-fullscreen-btn.dark-theme {
background: rgba(50, 50, 50, 0.95);
border: 1px solid rgba(255, 255, 255, 0.3);
color: #e0e0e0;
}
.mermaid-fullscreen-btn.dark-theme:hover {
background: rgba(60, 60, 60, 1);
box-shadow: 0 3px 10px rgba(255, 255, 255, 0.2);
}
.mermaid-fullscreen-modal {
display: none;
position: fixed !important;
top: 0 !important;
left: 0 !important;
width: 95vw;
height: 100vh;
background: rgba(255, 255, 255, 0.98);
z-index: 9999;
padding: 20px;
overflow: auto;
}
.mermaid-fullscreen-modal.dark-theme {
background: rgba(0, 0, 0, 0.98);
}
.mermaid-fullscreen-modal.active {
display: flex;
align-items: center;
justify-content: center;
}
.mermaid-container-fullscreen {
position: relative;
width: 95vw;
height: 90vh;
max-width: 95vw;
max-height: 90vh;
background: white;
border-radius: 8px;
padding: 20px;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
overflow: auto;
display: flex;
align-items: center;
justify-content: center;
}
.mermaid-container-fullscreen.dark-theme {
background: #1a1a1a;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.8);
}
.mermaid-container-fullscreen pre.mermaid {
width: 100%;
height: 100%;
display: flex;
align-items: center;
justify-content: center;
}
.mermaid-container-fullscreen .mermaid svg {
height: 100% !important;
width: 100% !important;
cursor: grab;
}
.mermaid-fullscreen-close {
position: fixed !important;
top: 20px !important;
right: 20px !important;
width: 40px;
height: 40px;
background: rgba(255, 255, 255, 0.95);
border: 1px solid rgba(0, 0, 0, 0.2);
border-radius: 50%;
cursor: pointer;
z-index: 10000;
display: flex;
align-items: center;
justify-content: center;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
transition: all 0.2s;
font-size: 24px;
line-height: 1;
color: #333;
}
.mermaid-fullscreen-close:hover {
background: white;
box-shadow: 0 6px 16px rgba(0, 0, 0, 0.4);
transform: scale(1.1);
}
.mermaid-fullscreen-close.dark-theme {
background: rgba(50, 50, 50, 0.95);
border: 1px solid rgba(255, 255, 255, 0.2);
color: #e0e0e0;
}
.mermaid-fullscreen-close.dark-theme:hover {
background: rgba(60, 60, 60, 1);
box-shadow: 0 6px 16px rgba(255, 255, 255, 0.2);
}
.mermaid-fullscreen-modal .mermaid-fullscreen-btn {
display: none !important;
}`;
document.head.appendChild(fullscreenStyle);
// Detect if page has dark background
const isDarkTheme = () => {
const bgColor = window.getComputedStyle(document.body).backgroundColor;
const match = bgColor.match(/rgb\((\d+),\s*(\d+),\s*(\d+)/);
if (match) {
const r = parseInt(match[1]);
const g = parseInt(match[2]);
const b = parseInt(match[3]);
const brightness = (r * 299 + g * 587 + b * 114) / 1000;
return brightness < 128;
}
return false;
};
const load = async () => {
await mermaid.run();
const all_mermaids = document.querySelectorAll(".mermaid");
const mermaids_processed = document.querySelectorAll(".mermaid[data-processed='true']");
if ("False" === "True") {
const mermaids_to_add_zoom = -1 === -1 ? all_mermaids.length : -1;
if(mermaids_to_add_zoom > 0) {
var svgs = d3.selectAll("");
if(all_mermaids.length !== mermaids_processed.length) {
setTimeout(load, 200);
return;
} else if(svgs.size() !== mermaids_to_add_zoom) {
setTimeout(load, 200);
return;
} else {
svgs.each(function() {
var svg = d3.select(this);
svg.html("<g class='wrapper'>" + svg.html() + "</g>");
var inner = svg.select("g");
var zoom = d3.zoom().on("zoom", function(event) {
inner.attr("transform", event.transform);
});
svg.call(zoom);
});
}
}
} else if(all_mermaids.length !== mermaids_processed.length) {
// Wait for mermaid to process all diagrams
setTimeout(load, 200);
return;
}
const darkTheme = isDarkTheme();
// Stop here if not adding fullscreen capability
if ("True" !== "True") return;
const modal = document.createElement('div');
modal.className = 'mermaid-fullscreen-modal' + (darkTheme ? ' dark-theme' : '');
modal.setAttribute('role', 'dialog');
modal.setAttribute('aria-modal', 'true');
modal.setAttribute('aria-label', 'Fullscreen diagram viewer');
modal.innerHTML = `
<button class="mermaid-fullscreen-close${darkTheme ? ' dark-theme' : ''}" aria-label="Close fullscreen">✕</button>
<div class="mermaid-container-fullscreen${darkTheme ? ' dark-theme' : ''}"></div>
`;
document.body.appendChild(modal);
const modalContent = modal.querySelector('.mermaid-container-fullscreen');
const closeBtn = modal.querySelector('.mermaid-fullscreen-close');
let previousScrollOffset = [window.scrollX, window.scrollY];
const closeModal = () => {
modal.classList.remove('active');
modalContent.innerHTML = '';
document.body.style.overflow = ''
window.scrollTo({left: previousScrollOffset[0], top: previousScrollOffset[1], behavior: 'instant'});
};
closeBtn.addEventListener('click', closeModal);
modal.addEventListener('click', (e) => {
if (e.target === modal) closeModal();
});
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape' && modal.classList.contains('active')) {
closeModal();
}
});
const allButtons = [];
document.querySelectorAll('.mermaid').forEach((mermaidDiv) => {
if (mermaidDiv.parentNode.classList.contains('mermaid-container') ||
mermaidDiv.closest('.mermaid-fullscreen-modal')) {
return;
}
const container = document.createElement('div');
container.className = 'mermaid-container';
mermaidDiv.parentNode.insertBefore(container, mermaidDiv);
container.appendChild(mermaidDiv);
const fullscreenBtn = document.createElement('button');
fullscreenBtn.className = 'mermaid-fullscreen-btn' + (darkTheme ? ' dark-theme' : '');
fullscreenBtn.setAttribute('aria-label', 'View diagram in fullscreen');
fullscreenBtn.textContent = '⛶';
fullscreenBtn.style.opacity = '50%';
// Calculate dynamic position based on diagram's margin and padding
const diagramStyle = window.getComputedStyle(mermaidDiv);
const marginTop = parseFloat(diagramStyle.marginTop) || 0;
const marginRight = parseFloat(diagramStyle.marginRight) || 0;
const paddingTop = parseFloat(diagramStyle.paddingTop) || 0;
const paddingRight = parseFloat(diagramStyle.paddingRight) || 0;
fullscreenBtn.style.top = `${marginTop + paddingTop + 4}px`;
fullscreenBtn.style.right = `${marginRight + paddingRight + 4}px`;
fullscreenBtn.addEventListener('click', () => {
previousScrollOffset = [window.scroll, window.scrollY];
const clone = mermaidDiv.cloneNode(true);
modalContent.innerHTML = '';
modalContent.appendChild(clone);
const svg = clone.querySelector('svg');
if (svg) {
svg.removeAttribute('width');
svg.removeAttribute('height');
svg.style.width = '100%';
svg.style.height = 'auto';
svg.style.maxWidth = '100%';
svg.style.sdisplay = 'block';
if ("False" === "True") {
setTimeout(() => {
const g = svg.querySelector('g');
if (g) {
var svgD3 = d3.select(svg);
svgD3.html("<g class='wrapper'>" + svgD3.html() + "</g>");
var inner = svgD3.select("g");
var zoom = d3.zoom().on("zoom", function(event) {
inner.attr("transform", event.transform);
});
svgD3.call(zoom);
}
}, 100);
}
}
modal.classList.add('active');
document.body.style.overflow = 'hidden';
});
container.appendChild(fullscreenBtn);
allButtons.push(fullscreenBtn);
});
// Update theme classes when theme changes
const updateTheme = () => {
const dark = isDarkTheme();
allButtons.forEach(btn => {
if (dark) {
btn.classList.add('dark-theme');
} else {
btn.classList.remove('dark-theme');
}
});
if (dark) {
modal.classList.add('dark-theme');
modalContent.classList.add('dark-theme');
closeBtn.classList.add('dark-theme');
} else {
modal.classList.remove('dark-theme');
modalContent.classList.remove('dark-theme');
closeBtn.classList.remove('dark-theme');
}
};
// Watch for theme changes
const observer = new MutationObserver(updateTheme);
observer.observe(document.documentElement, {
attributes: true,
attributeFilter: ['class', 'style', 'data-theme']
});
observer.observe(document.body, {
attributes: true,
attributeFilter: ['class', 'style']
});
};
window.addEventListener("load", load);
</script>
<script>DOCUMENTATION_OPTIONS.pagename = 'chapters/milestones';</script>
<script src="../_static/ml-timeline.js?v=76e9b3e3"></script>
<script src="../_static/wip-banner.js?v=04a7e74d"></script>
<script src="../_static/marimo-badges.js?v=e6289128"></script>
<script src="../_static/sidebar-link.js?v=404b701b"></script>
<script src="../_static/hero-carousel.js?v=10341d2a"></script>
<script src="../_static/subscribe-modal.js?v=42919b64"></script>
<link rel="icon" href="../_static/favicon.svg"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Frequently Asked Questions" href="../faq.html" />
<link rel="prev" title="The Learning Journey: From Atoms to Intelligence" href="learning-journey.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<input type="checkbox"
class="sidebar-toggle"
id="pst-primary-sidebar-checkbox"/>
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
<input type="checkbox"
class="sidebar-toggle"
id="pst-secondary-sidebar-checkbox"/>
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
<div class="search-button__wrapper">
<div class="search-button__overlay"></div>
<div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
id="search-input"
placeholder="Search..."
aria-label="Search..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
</div>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<div class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<a class="navbar-brand logo" href="../intro.html">
<img src="../_static/logo-tinytorch.png" class="logo__image only-light" alt="Tiny🔥Torch - Home"/>
<script>document.write(`<img src="../_static/logo-tinytorch.png" class="logo__image only-dark" alt="Tiny🔥Torch - Home"/>`);</script>
</a></div>
<div class="sidebar-primary-item">
<script>
document.write(`
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script></div>
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
<div class="bd-toc-item navbar-nav active">
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🚀 Getting Started</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../getting-started.html">Complete Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏗 Foundation Tier (01-07)</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../tiers/foundation.html">📖 Tier Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/01_tensor_ABOUT.html">01. Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/02_activations_ABOUT.html">02. Activations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/03_layers_ABOUT.html">03. Layers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/04_losses_ABOUT.html">04. Losses</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/05_autograd_ABOUT.html">05. Autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/06_optimizers_ABOUT.html">06. Optimizers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/07_training_ABOUT.html">07. Training</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏛️ Architecture Tier (08-13)</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../tiers/architecture.html">📖 Tier Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/08_dataloader_ABOUT.html">08. DataLoader</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/09_spatial_ABOUT.html">09. Convolutions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/10_tokenization_ABOUT.html">10. Tokenization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/11_embeddings_ABOUT.html">11. Embeddings</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/12_attention_ABOUT.html">12. Attention</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/13_transformers_ABOUT.html">13. Transformers</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">⏱️ Optimization Tier (14-19)</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../tiers/optimization.html">📖 Tier Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/14_profiling_ABOUT.html">14. Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/15_quantization_ABOUT.html">15. Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/16_compression_ABOUT.html">16. Compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/17_memoization_ABOUT.html">17. Memoization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/18_acceleration_ABOUT.html">18. Acceleration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/19_benchmarking_ABOUT.html">19. Benchmarking</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏅 Capstone Competition</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../tiers/olympics.html">📖 Competition Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../modules/20_capstone_ABOUT.html">20. Torch Olympics</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🧭 Course Orientation</span></p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="00-introduction.html">Course Structure</a></li>
<li class="toctree-l1"><a class="reference internal" href="../prerequisites.html">Prerequisites &amp; Resources</a></li>
<li class="toctree-l1"><a class="reference internal" href="learning-journey.html">Learning Journey</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Historical Milestones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🛠️ TITO CLI Reference</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../tito/overview.html">Command Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tito/modules.html">Module Workflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tito/milestones.html">Milestone System</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tito/data.html">Progress &amp; Data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tito/troubleshooting.html">Troubleshooting</a></li>
<li class="toctree-l1"><a class="reference internal" href="../datasets.html">Datasets Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🤝 Community</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../community.html">Ecosystem</a></li>
<li class="toctree-l1"><a class="reference internal" href="../resources.html">Learning Resources</a></li>
<li class="toctree-l1"><a class="reference internal" href="../credits.html">Credits &amp; Acknowledgments</a></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
</div>
<div id="rtd-footer-container"></div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="sbt-scroll-pixel-helper"></div>
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="fa-solid fa-bars"></span>
</button></div>
</div>
<div class="header-article-items__end">
<div class="header-article-item">
<div class="article-header-buttons">
<div class="dropdown dropdown-download-buttons">
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
<i class="fas fa-download"></i>
</button>
<ul class="dropdown-menu">
<li><a href="../_sources/chapters/milestones.md" target="_blank"
class="btn btn-sm btn-download-source-button dropdown-item"
title="Download source file"
data-bs-placement="left" data-bs-toggle="tooltip"
>
<span class="btn__icon-container">
<i class="fas fa-file"></i>
</span>
<span class="btn__text-container">.md</span>
</a>
</li>
<li>
<button onclick="window.print()"
class="btn btn-sm btn-download-pdf-button dropdown-item"
title="Print to PDF"
data-bs-placement="left" data-bs-toggle="tooltip"
>
<span class="btn__icon-container">
<i class="fas fa-file-pdf"></i>
</span>
<span class="btn__text-container">.pdf</span>
</button>
</li>
</ul>
</div>
<button onclick="toggleFullScreen()"
class="btn btn-sm btn-fullscreen-button"
title="Fullscreen mode"
data-bs-placement="bottom" data-bs-toggle="tooltip"
>
<span class="btn__icon-container">
<i class="fas fa-expand"></i>
</span>
</button>
<script>
document.write(`
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
</button>
`);
</script>
<script>
document.write(`
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
</button>
`);
</script>
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="fa-solid fa-list"></span>
</button>
</div></div>
</div>
</div>
</div>
<div id="jb-print-docs-body" class="onlyprint">
<h1>Journey Through ML History</h1>
<!-- Table of contents -->
<div id="print-main-content">
<div id="jb-print-toc">
<div>
<h2> Contents </h2>
</div>
<nav aria-label="Page">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-are-milestones">What Are Milestones?</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-this-approach">Why This Approach?</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#two-dimensions-of-your-progress">Two Dimensions of Your Progress</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#pedagogical-dimension-acts-what-youre-learning">Pedagogical Dimension (Acts): What You’re LEARNING</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#historical-dimension-milestones-what-you-can-build">Historical Dimension (Milestones): What You CAN Build</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-they-connect">How They Connect</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-timeline">The Timeline</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#perceptron-1957-rosenblatt">01. Perceptron (1957) - Rosenblatt</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#xor-crisis-1969-minsky-papert">02. XOR Crisis (1969) - Minsky &amp; Papert</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mlp-revival-1986-backpropagation-era">03. MLP Revival (1986) - Backpropagation Era</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#cnn-revolution-1998-lecuns-breakthrough">04. CNN Revolution (1998) - LeCun’s Breakthrough</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#transformer-era-2017-attention-revolution">05. Transformer Era (2017) - Attention Revolution</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#torch-olympics-era-2018-the-optimization-revolution">06. Torch Olympics Era (2018) - The Optimization Revolution</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-philosophy">Learning Philosophy</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-capability-building">Progressive Capability Building</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-engineering-progression">Systems Engineering Progression</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#how-to-use-milestones">How to Use Milestones</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-prerequisites">1. Complete Prerequisites</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#run-the-milestone">2. Run the Milestone</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#understand-the-systems">3. Understand the Systems</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reflect-and-compare">4. Reflect and Compare</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#quick-reference">Quick Reference</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#milestone-prerequisites">Milestone Prerequisites</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-each-milestone-proves">What Each Milestone Proves</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-learning">Further Learning</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-this-matters">Why This Matters</a></li>
</ul>
</nav>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section id="journey-through-ml-history">
<h1>Journey Through ML History<a class="headerlink" href="#journey-through-ml-history" title="Link to this heading">#</a></h1>
<p><strong>Experience the evolution of AI by rebuilding history’s most important breakthroughs with YOUR TinyTorch implementations.</strong></p>
<hr class="docutils" />
<section id="what-are-milestones">
<h2>What Are Milestones?<a class="headerlink" href="#what-are-milestones" title="Link to this heading">#</a></h2>
<p>Milestones are <strong>proof-of-mastery demonstrations</strong> that showcase what you can build after completing specific modules. Each milestone recreates a historically significant ML achievement using YOUR implementations.</p>
<section id="why-this-approach">
<h3>Why This Approach?<a class="headerlink" href="#why-this-approach" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p><strong>Deep Understanding</strong>: Experience the actual challenges researchers faced</p></li>
<li><p><strong>Progressive Learning</strong>: Each milestone builds on previous foundations</p></li>
<li><p><strong>Real Achievements</strong>: Not toy examples - these are historically significant breakthroughs</p></li>
<li><p><strong>Systems Thinking</strong>: Understand WHY each innovation mattered for ML systems</p></li>
</ul>
</section>
</section>
<hr class="docutils" />
<section id="two-dimensions-of-your-progress">
<h2>Two Dimensions of Your Progress<a class="headerlink" href="#two-dimensions-of-your-progress" title="Link to this heading">#</a></h2>
<p>As you build TinyTorch, you’re progressing along <strong>TWO dimensions simultaneously</strong>:</p>
<section id="pedagogical-dimension-acts-what-youre-learning">
<h3>Pedagogical Dimension (Acts): What You’re LEARNING<a class="headerlink" href="#pedagogical-dimension-acts-what-youre-learning" title="Link to this heading">#</a></h3>
<p><strong>Act I (01-04)</strong>: Building atomic components - mathematical foundations
<strong>Act II (05-07)</strong>: The gradient revolution - systems that learn
<strong>Act III (08-09)</strong>: Real-world complexity - data and scale
<strong>Act IV (10-13)</strong>: Sequential intelligence - language understanding
<strong>Act V (14-19)</strong>: Production systems - optimization and deployment
<strong>Act VI (20)</strong>: Complete integration - unified AI systems</p>
<p>See <a class="reference internal" href="learning-journey.html"><span class="std std-doc">The Learning Journey</span></a> for the complete pedagogical narrative explaining WHY modules flow this way.</p>
</section>
<section id="historical-dimension-milestones-what-you-can-build">
<h3>Historical Dimension (Milestones): What You CAN Build<a class="headerlink" href="#historical-dimension-milestones-what-you-can-build" title="Link to this heading">#</a></h3>
<p><strong>1957: Perceptron</strong> - Binary classification
<strong>1969: XOR</strong> - Non-linear learning
<strong>1986: MLP</strong> - Multi-class vision
<strong>1998: CNN</strong> - Spatial intelligence
<strong>2017: Transformers</strong> - Language generation
<strong>2018: Torch Olympics</strong> - Production optimization</p>
</section>
<section id="how-they-connect">
<h3>How They Connect<a class="headerlink" href="#how-they-connect" title="Link to this heading">#</a></h3>
<pre class="mermaid">
graph TB
subgraph &quot;Pedagogical Acts (What You're Learning)&quot;
A1[&quot;Act I: Foundation&lt;br/&gt;Modules 01-04&lt;br/&gt;Atomic Components&quot;]
A2[&quot;Act II: Learning&lt;br/&gt;Modules 05-07&lt;br/&gt;Gradient Revolution&quot;]
A3[&quot;Act III: Data &amp; Scale&lt;br/&gt;Modules 08-09&lt;br/&gt;Real-World Complexity&quot;]
A4[&quot;Act IV: Language&lt;br/&gt;Modules 10-13&lt;br/&gt;Sequential Intelligence&quot;]
A5[&quot;Act V: Production&lt;br/&gt;Modules 14-19&lt;br/&gt;Optimization&quot;]
A6[&quot;Act VI: Integration&lt;br/&gt;Module 20&lt;br/&gt;Complete Systems&quot;]
end
subgraph &quot;Historical Milestones (What You Can Build)&quot;
M1[&quot;1957: Perceptron&lt;br/&gt;Binary Classification&quot;]
M2[&quot;1969: XOR Crisis&lt;br/&gt;Non-linear Learning&quot;]
M3[&quot;1986: MLP&lt;br/&gt;Multi-class Vision&lt;br/&gt;95%+ MNIST&quot;]
M4[&quot;1998: CNN&lt;br/&gt;Spatial Intelligence&lt;br/&gt;75%+ CIFAR-10&quot;]
M5[&quot;2017: Transformers&lt;br/&gt;Language Generation&quot;]
M6[&quot;2018: Torch Olympics&lt;br/&gt;Production Speed&quot;]
end
A1 --&gt; M1
A2 --&gt; M2
A2 --&gt; M3
A3 --&gt; M4
A4 --&gt; M5
A5 --&gt; M6
style A1 fill:#e3f2fd
style A2 fill:#fff8e1
style A3 fill:#e8f5e9
style A4 fill:#f3e5f5
style A5 fill:#fce4ec
style A6 fill:#fff3e0
style M1 fill:#ffcdd2
style M2 fill:#f8bbd0
style M3 fill:#e1bee7
style M4 fill:#d1c4e9
style M5 fill:#c5cae9
style M6 fill:#bbdefb
</pre><div class="pst-scrollable-table-container"><table class="table">
<thead>
<tr class="row-odd"><th class="head"><p>Learning Act</p></th>
<th class="head"><p>Unlocked Milestone</p></th>
<th class="head"><p>Proof of Mastery</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><strong>Act I: Foundation (01-04)</strong></p></td>
<td><p>1957 Perceptron</p></td>
<td><p>Your Linear layer recreates history</p></td>
</tr>
<tr class="row-odd"><td><p><strong>Act II: Learning (05-07)</strong></p></td>
<td><p>1969 XOR + 1986 MLP</p></td>
<td><p>Your autograd enables training (95%+ MNIST)</p></td>
</tr>
<tr class="row-even"><td><p><strong>Act III: Data &amp; Scale (08-09)</strong></p></td>
<td><p>1998 CNN</p></td>
<td><p>Your Conv2d achieves 75%+ on CIFAR-10</p></td>
</tr>
<tr class="row-odd"><td><p><strong>Act IV: Language (10-13)</strong></p></td>
<td><p>2017 Transformers</p></td>
<td><p>Your attention generates coherent text</p></td>
</tr>
<tr class="row-even"><td><p><strong>Act V: Production (14-18)</strong></p></td>
<td><p>2018 Torch Olympics</p></td>
<td><p>Your optimizations achieve production speed</p></td>
</tr>
<tr class="row-odd"><td><p><strong>Act VI: Integration (19-20)</strong></p></td>
<td><p>Benchmarking + Capstone</p></td>
<td><p>Your complete framework competes</p></td>
</tr>
</tbody>
</table>
</div>
<p><strong>Understanding Both Dimensions</strong>: The <strong>Acts</strong> explain WHY you’re building each component (pedagogical progression). The <strong>Milestones</strong> prove WHAT you’ve built works (historical validation). Together, they show you’re not just completing exercises - you’re building something real.</p>
</section>
</section>
<hr class="docutils" />
<section id="the-timeline">
<h2>The Timeline<a class="headerlink" href="#the-timeline" title="Link to this heading">#</a></h2>
<pre class="mermaid">
timeline
title Journey Through ML History
1957 : Perceptron : Binary classification with gradient descent
1969 : XOR Crisis : Hidden layers solve non-linear problems
1986 : MLP Revival : Backpropagation enables deep learning
1998 : CNN Era : Spatial intelligence for computer vision
2017 : Transformers : Attention revolutionizes language AI
2018 : Torch Olympics : Production benchmarking and optimization
</pre><section id="perceptron-1957-rosenblatt">
<h3>01. Perceptron (1957) - Rosenblatt<a class="headerlink" href="#perceptron-1957-rosenblatt" title="Link to this heading">#</a></h3>
<p><strong>After Modules 02-04</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Input → Linear → Sigmoid → Output
</pre></div>
</div>
<p><strong>The Beginning</strong>: The first trainable neural network. Frank Rosenblatt proved machines could learn from data.</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Binary classification with gradient descent</p></li>
<li><p>Simple but revolutionary architecture</p></li>
<li><p>YOUR Linear layer recreates history</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: O(n) parameters</p></li>
<li><p>Compute: O(n) operations</p></li>
<li><p>Limitation: Only linearly separable problems</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/01_1957_perceptron
python<span class="w"> </span>01_rosenblatt_forward.py<span class="w"> </span><span class="c1"># See the problem (random weights)</span>
python<span class="w"> </span>02_rosenblatt_trained.py<span class="w"> </span><span class="c1"># See the solution (trained)</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: ~50% (untrained) → 95%+ (trained) accuracy</p>
</section>
<hr class="docutils" />
<section id="xor-crisis-1969-minsky-papert">
<h3>02. XOR Crisis (1969) - Minsky &amp; Papert<a class="headerlink" href="#xor-crisis-1969-minsky-papert" title="Link to this heading">#</a></h3>
<p><strong>After Modules 02-06</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Input → Linear → ReLU → Linear → Output
</pre></div>
</div>
<p><strong>The Challenge</strong>: Minsky proved perceptrons couldn’t solve XOR. This crisis nearly ended AI research.</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Hidden layers enable non-linear solutions</p></li>
<li><p>Multi-layer networks break through limitations</p></li>
<li><p>YOUR autograd makes it possible</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: O(n²) with hidden layers</p></li>
<li><p>Compute: O(n²) operations</p></li>
<li><p>Breakthrough: Hidden representations</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/02_1969_xor
python<span class="w"> </span>01_xor_crisis.py<span class="w"> </span><span class="c1"># Watch it fail (loss stuck at 0.69)</span>
python<span class="w"> </span>02_xor_solved.py<span class="w"> </span><span class="c1"># Hidden layers solve it!</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: 50% (single layer) → 100% (multi-layer) on XOR</p>
</section>
<hr class="docutils" />
<section id="mlp-revival-1986-backpropagation-era">
<h3>03. MLP Revival (1986) - Backpropagation Era<a class="headerlink" href="#mlp-revival-1986-backpropagation-era" title="Link to this heading">#</a></h3>
<p><strong>After Modules 02-08</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Images → Flatten → Linear → ReLU → Linear → ReLU → Linear → Classes
</pre></div>
</div>
<p><strong>The Revolution</strong>: Backpropagation enabled training deep networks on real datasets like MNIST.</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Multi-class digit recognition</p></li>
<li><p>Complete training pipelines</p></li>
<li><p>YOUR optimizers achieve 95%+ accuracy</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: ~100K parameters for MNIST</p></li>
<li><p>Compute: Dense matrix operations</p></li>
<li><p>Architecture: Multi-layer feature learning</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/03_1986_mlp
python<span class="w"> </span>01_rumelhart_tinydigits.py<span class="w"> </span><span class="c1"># 8x8 digits (quick)</span>
python<span class="w"> </span>02_rumelhart_mnist.py<span class="w"> </span><span class="c1"># Full MNIST</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: 95%+ accuracy on MNIST</p>
</section>
<hr class="docutils" />
<section id="cnn-revolution-1998-lecuns-breakthrough">
<h3>04. CNN Revolution (1998) - LeCun’s Breakthrough<a class="headerlink" href="#cnn-revolution-1998-lecuns-breakthrough" title="Link to this heading">#</a></h3>
<p><strong>After Modules 02-09</strong> • <strong>🎯 North Star Achievement</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Images → Conv → ReLU → Pool → Conv → ReLU → Pool → Flatten → Linear → Classes
</pre></div>
</div>
<p><strong>The Game-Changer</strong>: CNNs exploit spatial structure for computer vision. This enabled modern AI.</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Convolutional feature extraction</p></li>
<li><p>Natural image classification (CIFAR-10)</p></li>
<li><p>YOUR Conv2d + MaxPool2d unlock spatial intelligence</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: ~1M parameters (weight sharing reduces vs dense)</p></li>
<li><p>Compute: Convolution is intensive but parallelizable</p></li>
<li><p>Architecture: Local connectivity + translation invariance</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/04_1998_cnn
python<span class="w"> </span>01_lecun_tinydigits.py<span class="w"> </span><span class="c1"># Spatial features on digits</span>
python<span class="w"> </span>02_lecun_cifar10.py<span class="w"> </span><span class="c1"># CIFAR-10 @ 75%+ accuracy</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: <strong>75%+ accuracy on CIFAR-10</strong> ✨</p>
</section>
<hr class="docutils" />
<section id="transformer-era-2017-attention-revolution">
<h3>05. Transformer Era (2017) - Attention Revolution<a class="headerlink" href="#transformer-era-2017-attention-revolution" title="Link to this heading">#</a></h3>
<p><strong>After Modules 02-13</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Tokens → Embeddings → Attention → FFN → ... → Attention → Output
</pre></div>
</div>
<p><strong>The Modern Era</strong>: Transformers + attention launched the LLM revolution (GPT, BERT, ChatGPT).</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Self-attention mechanisms</p></li>
<li><p>Autoregressive text generation</p></li>
<li><p>YOUR attention implementation generates language</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: O(n²) attention requires careful management</p></li>
<li><p>Compute: Highly parallelizable</p></li>
<li><p>Architecture: Long-range dependencies</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/05_2017_transformer
python<span class="w"> </span>01_vaswani_generation.py<span class="w"> </span><span class="c1"># Q&amp;A generation with TinyTalks</span>
python<span class="w"> </span>02_vaswani_dialogue.py<span class="w"> </span><span class="c1"># Multi-turn dialogue</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: Loss &lt; 1.5, coherent responses to questions</p>
</section>
<hr class="docutils" />
<section id="torch-olympics-era-2018-the-optimization-revolution">
<h3>06. Torch Olympics Era (2018) - The Optimization Revolution<a class="headerlink" href="#torch-olympics-era-2018-the-optimization-revolution" title="Link to this heading">#</a></h3>
<p><strong>After Modules 14-18</strong></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Profile → Compress → Accelerate
</pre></div>
</div>
<p><strong>The Turning Point</strong>: As models grew larger, MLCommons’ Torch Olympics (2018) established systematic optimization as a discipline - profiling, compression, and acceleration became essential for deployment.</p>
<p><strong>What You’ll Build</strong>:</p>
<ul class="simple">
<li><p>Performance profiling and bottleneck analysis</p></li>
<li><p>Model compression (quantization + pruning)</p></li>
<li><p>Inference acceleration (KV-cache + batching)</p></li>
</ul>
<p><strong>Systems Insights</strong>:</p>
<ul class="simple">
<li><p>Memory: 4-16× compression through quantization/pruning</p></li>
<li><p>Speed: 12-40× faster generation with KV-cache + batching</p></li>
<li><p>Workflow: Systematic “measure → optimize → validate” methodology</p></li>
</ul>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/06_2018_mlperf
python<span class="w"> </span>01_baseline_profile.py<span class="w"> </span><span class="c1"># Find bottlenecks</span>
python<span class="w"> </span>02_compression.py<span class="w"> </span><span class="c1"># Reduce size (quantize + prune)</span>
python<span class="w"> </span>03_generation_opts.py<span class="w"> </span><span class="c1"># Speed up inference (cache + batch)</span>
</pre></div>
</div>
<p><strong>Expected Results</strong>: 8-16× smaller models, 12-40× faster inference</p>
</section>
</section>
<hr class="docutils" />
<section id="learning-philosophy">
<h2>Learning Philosophy<a class="headerlink" href="#learning-philosophy" title="Link to this heading">#</a></h2>
<section id="progressive-capability-building">
<h3>Progressive Capability Building<a class="headerlink" href="#progressive-capability-building" title="Link to this heading">#</a></h3>
<div class="pst-scrollable-table-container"><table class="table">
<thead>
<tr class="row-odd"><th class="head"><p>Stage</p></th>
<th class="head"><p>Era</p></th>
<th class="head"><p>Capability</p></th>
<th class="head"><p>Your Tools</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><strong>1957</strong></p></td>
<td><p>Foundation</p></td>
<td><p>Binary classification</p></td>
<td><p>Linear + Sigmoid</p></td>
</tr>
<tr class="row-odd"><td><p><strong>1969</strong></p></td>
<td><p>Depth</p></td>
<td><p>Non-linear problems</p></td>
<td><p>Hidden layers + Autograd</p></td>
</tr>
<tr class="row-even"><td><p><strong>1986</strong></p></td>
<td><p>Scale</p></td>
<td><p>Multi-class vision</p></td>
<td><p>Optimizers + Training</p></td>
</tr>
<tr class="row-odd"><td><p><strong>1998</strong></p></td>
<td><p>Structure</p></td>
<td><p>Spatial understanding</p></td>
<td><p>Conv2d + Pooling</p></td>
</tr>
<tr class="row-even"><td><p><strong>2017</strong></p></td>
<td><p>Attention</p></td>
<td><p>Sequence modeling</p></td>
<td><p>Transformers + Attention</p></td>
</tr>
<tr class="row-odd"><td><p><strong>2018</strong></p></td>
<td><p>Optimization</p></td>
<td><p>Production deployment</p></td>
<td><p>Profiling + Compression + Acceleration</p></td>
</tr>
</tbody>
</table>
</div>
</section>
<section id="systems-engineering-progression">
<h3>Systems Engineering Progression<a class="headerlink" href="#systems-engineering-progression" title="Link to this heading">#</a></h3>
<p>Each milestone teaches critical systems thinking:</p>
<ol class="arabic simple">
<li><p><strong>Memory Management</strong>: From O(n) → O(n²) → O(n²) with optimizations</p></li>
<li><p><strong>Computational Trade-offs</strong>: Accuracy vs efficiency</p></li>
<li><p><strong>Architectural Patterns</strong>: How structure enables capability</p></li>
<li><p><strong>Production Deployment</strong>: What it takes to scale</p></li>
</ol>
</section>
</section>
<hr class="docutils" />
<section id="how-to-use-milestones">
<h2>How to Use Milestones<a class="headerlink" href="#how-to-use-milestones" title="Link to this heading">#</a></h2>
<section id="complete-prerequisites">
<h3>1. Complete Prerequisites<a class="headerlink" href="#complete-prerequisites" title="Link to this heading">#</a></h3>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Check which modules you&#39;ve completed</span>
tito<span class="w"> </span>checkpoint<span class="w"> </span>status
<span class="c1"># Complete required modules</span>
tito<span class="w"> </span>module<span class="w"> </span><span class="nb">complete</span><span class="w"> </span>02_tensor
tito<span class="w"> </span>module<span class="w"> </span><span class="nb">complete</span><span class="w"> </span>03_activations
<span class="c1"># ... and so on</span>
</pre></div>
</div>
</section>
<section id="run-the-milestone">
<h3>2. Run the Milestone<a class="headerlink" href="#run-the-milestone" title="Link to this heading">#</a></h3>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/01_1957_perceptron
python<span class="w"> </span>02_rosenblatt_trained.py
</pre></div>
</div>
</section>
<section id="understand-the-systems">
<h3>3. Understand the Systems<a class="headerlink" href="#understand-the-systems" title="Link to this heading">#</a></h3>
<p>Each milestone includes:</p>
<ul class="simple">
<li><p>📊 <strong>Memory profiling</strong>: See actual memory usage</p></li>
<li><p>⚡ <strong>Performance metrics</strong>: FLOPs, parameters, timing</p></li>
<li><p>🧠 <strong>Architectural analysis</strong>: Why this design matters</p></li>
<li><p>📈 <strong>Scaling insights</strong>: How performance changes with size</p></li>
</ul>
</section>
<section id="reflect-and-compare">
<h3>4. Reflect and Compare<a class="headerlink" href="#reflect-and-compare" title="Link to this heading">#</a></h3>
<p><strong>Questions to ask:</strong></p>
<ul class="simple">
<li><p>How does this compare to modern architectures?</p></li>
<li><p>What were the computational constraints in that era?</p></li>
<li><p>How would you optimize this for production?</p></li>
<li><p>What patterns appear in PyTorch/TensorFlow?</p></li>
</ul>
</section>
</section>
<hr class="docutils" />
<section id="quick-reference">
<h2>Quick Reference<a class="headerlink" href="#quick-reference" title="Link to this heading">#</a></h2>
<section id="milestone-prerequisites">
<h3>Milestone Prerequisites<a class="headerlink" href="#milestone-prerequisites" title="Link to this heading">#</a></h3>
<div class="pst-scrollable-table-container"><table class="table">
<thead>
<tr class="row-odd"><th class="head"><p>Milestone</p></th>
<th class="head"><p>After Module</p></th>
<th class="head"><p>Key Requirements</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>01. Perceptron (1957)</p></td>
<td><p>04</p></td>
<td><p>Tensor, Activations, Layers</p></td>
</tr>
<tr class="row-odd"><td><p>02. XOR (1969)</p></td>
<td><p>06</p></td>
<td><p>+ Losses, Autograd</p></td>
</tr>
<tr class="row-even"><td><p>03. MLP (1986)</p></td>
<td><p>08</p></td>
<td><p>+ Optimizers, Training</p></td>
</tr>
<tr class="row-odd"><td><p>04. CNN (1998)</p></td>
<td><p>09</p></td>
<td><p>+ Spatial, DataLoader</p></td>
</tr>
<tr class="row-even"><td><p>05. Transformer (2017)</p></td>
<td><p>13</p></td>
<td><p>+ Tokenization, Embeddings, Attention</p></td>
</tr>
<tr class="row-odd"><td><p>06. Torch Olympics (2018)</p></td>
<td><p>18</p></td>
<td><p>+ Profiling, Quantization, Compression, Memoization, Acceleration</p></td>
</tr>
</tbody>
</table>
</div>
</section>
<section id="what-each-milestone-proves">
<h3>What Each Milestone Proves<a class="headerlink" href="#what-each-milestone-proves" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p><strong>Your implementations work</strong> - Not just toy code</p></li>
<li><p><strong>Historical significance</strong> - These breakthroughs shaped modern AI</p></li>
<li><p><strong>Systems understanding</strong> - You know memory, compute, scaling</p></li>
<li><p><strong>Production relevance</strong> - Patterns used in real ML frameworks</p></li>
</ul>
</section>
</section>
<hr class="docutils" />
<section id="further-learning">
<h2>Further Learning<a class="headerlink" href="#further-learning" title="Link to this heading">#</a></h2>
<p>After completing milestones, explore:</p>
<ul class="simple">
<li><p><strong>Torch Olympics Competition</strong>: Optimize your implementations</p></li>
<li><p><strong>Leaderboard</strong>: Compare with other students</p></li>
<li><p><strong>Capstone Projects</strong>: Build your own ML applications</p></li>
<li><p><strong>Research Papers</strong>: Read the original papers for each milestone</p></li>
</ul>
</section>
<hr class="docutils" />
<section id="why-this-matters">
<h2>Why This Matters<a class="headerlink" href="#why-this-matters" title="Link to this heading">#</a></h2>
<p><strong>Most courses teach you to USE frameworks.</strong><br />
<strong>TinyTorch teaches you to UNDERSTAND them.</strong></p>
<p>By rebuilding ML history, you gain:</p>
<ul class="simple">
<li><p>🧠 Deep intuition for how neural networks work</p></li>
<li><p>🔧 Systems thinking for production ML</p></li>
<li><p>🏆 Portfolio projects demonstrating mastery</p></li>
<li><p>💼 Preparation for ML systems engineering roles</p></li>
</ul>
<hr class="docutils" />
<p><strong>Ready to start your journey through ML history?</strong></p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>milestones/01_1957_perceptron
python<span class="w"> </span>02_rosenblatt_trained.py
</pre></div>
</div>
<p><strong>Build the future by understanding the past.</strong> 🚀</p>
</section>
</section>
<script type="text/x-thebe-config">
{
requestKernel: true,
binderOptions: {
repo: "binder-examples/jupyter-stacks-datascience",
ref: "master",
},
codeMirrorConfig: {
theme: "abcdef",
mode: "python"
},
kernelOptions: {
name: "python3",
path: "./chapters"
},
predefinedOutput: true
}
</script>
<script>kernelName = 'python3'</script>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="learning-journey.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">The Learning Journey: From Atoms to Intelligence</p>
</div>
</a>
<a class="right-next"
href="../faq.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Frequently Asked Questions</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> Contents
</div>
<nav class="bd-toc-nav page-toc">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-are-milestones">What Are Milestones?</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#why-this-approach">Why This Approach?</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#two-dimensions-of-your-progress">Two Dimensions of Your Progress</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#pedagogical-dimension-acts-what-youre-learning">Pedagogical Dimension (Acts): What You’re LEARNING</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#historical-dimension-milestones-what-you-can-build">Historical Dimension (Milestones): What You CAN Build</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-they-connect">How They Connect</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-timeline">The Timeline</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#perceptron-1957-rosenblatt">01. Perceptron (1957) - Rosenblatt</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#xor-crisis-1969-minsky-papert">02. XOR Crisis (1969) - Minsky &amp; Papert</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mlp-revival-1986-backpropagation-era">03. MLP Revival (1986) - Backpropagation Era</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#cnn-revolution-1998-lecuns-breakthrough">04. CNN Revolution (1998) - LeCun’s Breakthrough</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#transformer-era-2017-attention-revolution">05. Transformer Era (2017) - Attention Revolution</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#torch-olympics-era-2018-the-optimization-revolution">06. Torch Olympics Era (2018) - The Optimization Revolution</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-philosophy">Learning Philosophy</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#progressive-capability-building">Progressive Capability Building</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-engineering-progression">Systems Engineering Progression</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#how-to-use-milestones">How to Use Milestones</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-prerequisites">1. Complete Prerequisites</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#run-the-milestone">2. Run the Milestone</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#understand-the-systems">3. Understand the Systems</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reflect-and-compare">4. Reflect and Compare</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#quick-reference">Quick Reference</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#milestone-prerequisites">Milestone Prerequisites</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#what-each-milestone-proves">What Each Milestone Proves</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-learning">Further Learning</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-this-matters">Why This Matters</a></li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
<div class="bd-footer-content__inner container">
<div class="footer-item">
<p class="component-author">
By Prof. Vijay Janapa Reddi (Harvard University)
</p>
</div>
<div class="footer-item">
<p class="copyright">
© Copyright 2025.
<br/>
</p>
</div>
<div class="footer-item">
</div>
<div class="footer-item">
</div>
</div>
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
<footer class="bd-footer">
</footer>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink