mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-05-24 20:56:43 -05:00
1263 lines
105 KiB
HTML
1263 lines
105 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="../" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>20. Capstone - Submission Infrastructure — Tiny🔥Torch</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="../_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="../_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/custom.css?v=009d37f4" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="../_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="../_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="../_static/copybutton.js?v=f281be69"></script>
|
||
<script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="../_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'modules/20_capstone_ABOUT';</script>
|
||
<script src="../_static/ml-timeline.js?v=76e9b3e3"></script>
|
||
<script src="../_static/wip-banner.js?v=04a7e74d"></script>
|
||
<script src="../_static/marimo-badges.js?v=e6289128"></script>
|
||
<script src="../_static/sidebar-link.js?v=404b701b"></script>
|
||
<script src="../_static/hero-carousel.js?v=10341d2a"></script>
|
||
<script src="../_static/subscribe-modal.js?v=42919b64"></script>
|
||
<link rel="icon" href="../_static/favicon.svg"/>
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
<link rel="next" title="Course Introduction: ML Systems Engineering Through Implementation" href="../chapters/00-introduction.html" />
|
||
<link rel="prev" title="🏅 Torch Olympics (Module 20)" href="../tiers/olympics.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="../search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search..."
|
||
aria-label="Search..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../_static/logo-tinytorch.png" class="logo__image only-light" alt="Tiny🔥Torch - Home"/>
|
||
<script>document.write(`<img src="../_static/logo-tinytorch.png" class="logo__image only-dark" alt="Tiny🔥Torch - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🚀 Getting Started</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../getting-started.html">Complete Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏗 Foundation Tier (01-07)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/foundation.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="01_tensor_ABOUT.html">01. Tensor</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="02_activations_ABOUT.html">02. Activations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="03_layers_ABOUT.html">03. Layers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="04_losses_ABOUT.html">04. Losses</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="05_autograd_ABOUT.html">05. Autograd</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="06_optimizers_ABOUT.html">06. Optimizers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="07_training_ABOUT.html">07. Training</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏛️ Architecture Tier (08-13)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/architecture.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="08_dataloader_ABOUT.html">08. DataLoader</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="09_spatial_ABOUT.html">09. Convolutions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="10_tokenization_ABOUT.html">10. Tokenization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="11_embeddings_ABOUT.html">11. Embeddings</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="12_attention_ABOUT.html">12. Attention</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="13_transformers_ABOUT.html">13. Transformers</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">⏱️ Optimization Tier (14-19)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/optimization.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="14_profiling_ABOUT.html">14. Profiling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="15_quantization_ABOUT.html">15. Quantization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="16_compression_ABOUT.html">16. Compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="17_memoization_ABOUT.html">17. Memoization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="18_acceleration_ABOUT.html">18. Acceleration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="19_benchmarking_ABOUT.html">19. Benchmarking</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏅 Capstone Competition</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/olympics.html">📖 Competition Overview</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">20. Torch Olympics</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🧭 Course Orientation</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/00-introduction.html">Course Structure</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../prerequisites.html">Prerequisites & Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/learning-journey.html">Learning Journey</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/milestones.html">Historical Milestones</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🛠️ TITO CLI Reference</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/overview.html">Command Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/modules.html">Module Workflow</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/milestones.html">Milestone System</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/data.html">Progress & Data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/troubleshooting.html">Troubleshooting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../datasets.html">Datasets Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🤝 Community</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../community.html">Ecosystem</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources.html">Learning Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../credits.html">Credits & Acknowledgments</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="../_sources/modules/20_capstone_ABOUT.md" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.md</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>20. Capstone - Submission Infrastructure</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#core-infrastructure-components">Core Infrastructure Components</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#benchmarkreport-class">1. BenchmarkReport Class</a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#submission-generation-function">2. Submission Generation Function</a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-optimization-workflow">3. Complete Optimization Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#connection-to-tinytorch-optimization-tier">Connection to TinyTorch Optimization Tier</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#enabling-future-competitions">Enabling Future Competitions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#optimization-workflow-integration">Optimization Workflow Integration</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#submission-reproducibility">Submission Reproducibility</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#statistical-measurement-validity">Statistical Measurement Validity</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#workflow-scalability">Workflow Scalability</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#competition-fairness">Competition Fairness</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-complete-the-optimization-tier">Ready to Complete the Optimization Tier?</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section id="capstone-submission-infrastructure">
|
||
<h1>20. Capstone - Submission Infrastructure<a class="headerlink" href="#capstone-submission-infrastructure" title="Link to this heading">#</a></h1>
|
||
<p><strong>OPTIMIZATION TIER CAPSTONE</strong> | Difficulty: ⭐⭐⭐⭐ (4/4) | Time: 5-8 hours</p>
|
||
<section id="overview">
|
||
<h2>Overview<a class="headerlink" href="#overview" title="Link to this heading">#</a></h2>
|
||
<p>Build professional submission infrastructure that brings together everything you’ve learned in the Optimization Tier. This capstone teaches you how to benchmark models using TinyTorch’s optimization APIs (Modules 14-19), generate standardized JSON submissions, and create shareable results for ML competitions.</p>
|
||
<p><strong>What You Learn</strong>: Complete optimization workflow from profiling to submission—how to use TinyTorch as a cohesive framework, not just individual modules. You’ll apply the optimization pipeline (Profile → Optimize → Benchmark → Submit) to demonstrate your framework’s capabilities with measurable, reproducible results.</p>
|
||
<p><strong>The Focus</strong>: Using TinyTorch’s APIs together in a professional workflow. This isn’t about building new optimization techniques—it’s about orchestrating existing tools to generate competition-ready submissions.</p>
|
||
</section>
|
||
<section id="learning-objectives">
|
||
<h2>Learning Objectives<a class="headerlink" href="#learning-objectives" title="Link to this heading">#</a></h2>
|
||
<p>By the end of this capstone, you will be able to:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Apply benchmarking tools systematically</strong>: Use Module 19’s <code class="docutils literal notranslate"><span class="pre">BenchmarkReport</span></code> class to measure model performance with statistical rigor</p></li>
|
||
<li><p><strong>Generate standardized submissions</strong>: Create MLPerf-inspired JSON submissions with system info, metrics, and reproducibility metadata</p></li>
|
||
<li><p><strong>Calculate improvement metrics</strong>: Compute normalized speedup, compression ratio, and accuracy delta for hardware-independent comparison</p></li>
|
||
<li><p><strong>Execute optimization workflows</strong>: Integrate profiling (M14), optimization techniques (M15-18), and benchmarking (M19) in complete pipeline</p></li>
|
||
<li><p><strong>Build competition infrastructure</strong>: Create submission formats that enable future challenges (Wake Vision, custom competitions)</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="build-use-reflect">
|
||
<h2>Build → Use → Reflect<a class="headerlink" href="#build-use-reflect" title="Link to this heading">#</a></h2>
|
||
<p>This capstone follows TinyTorch’s <strong>Build → Use → Reflect</strong> framework:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Build</strong>: Implement <code class="docutils literal notranslate"><span class="pre">BenchmarkReport</span></code> class, <code class="docutils literal notranslate"><span class="pre">generate_submission()</span></code> function, and standardized JSON schema with system metadata</p></li>
|
||
<li><p><strong>Use</strong>: Benchmark baseline and optimized models using TinyTorch’s optimization APIs; generate submissions comparing before/after performance</p></li>
|
||
<li><p><strong>Reflect</strong>: How do optimization techniques combine in practice? What makes submissions reproducible? How does standardized infrastructure enable fair competition?</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="implementation-guide">
|
||
<h2>Implementation Guide<a class="headerlink" href="#implementation-guide" title="Link to this heading">#</a></h2>
|
||
<section id="core-infrastructure-components">
|
||
<h3>Core Infrastructure Components<a class="headerlink" href="#core-infrastructure-components" title="Link to this heading">#</a></h3>
|
||
<p>The submission infrastructure implements three key systems:</p>
|
||
<section id="benchmarkreport-class">
|
||
<h4>1. BenchmarkReport Class<a class="headerlink" href="#benchmarkreport-class" title="Link to this heading">#</a></h4>
|
||
<p><strong>Comprehensive Performance Measurement</strong></p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">BenchmarkReport</span></code> class encapsulates all metrics needed for competition submissions:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">BenchmarkReport</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Benchmark report for model performance measurement.</span>
|
||
|
||
<span class="sd"> Collects and stores:</span>
|
||
<span class="sd"> - Model characteristics (parameters, size)</span>
|
||
<span class="sd"> - Performance metrics (accuracy, latency, throughput)</span>
|
||
<span class="sd"> - System information (hardware, software versions)</span>
|
||
<span class="sd"> - Timestamps for reproducibility</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model_name</span><span class="o">=</span><span class="s2">"model"</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">metrics</span> <span class="o">=</span> <span class="p">{}</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">system_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_system_info</span><span class="p">()</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">timestamp</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">benchmark_model</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">100</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Benchmark model with statistical rigor.</span>
|
||
|
||
<span class="sd"> Measures:</span>
|
||
<span class="sd"> - Parameter count and model size (MB)</span>
|
||
<span class="sd"> - Accuracy on test set</span>
|
||
<span class="sd"> - Latency with mean ± std (100 runs for statistics)</span>
|
||
<span class="sd"> - Throughput (samples/second)</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> Dict with all metrics for submission generation</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Count parameters</span>
|
||
<span class="n">param_count</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">count_parameters</span><span class="p">()</span>
|
||
<span class="n">model_size_mb</span> <span class="o">=</span> <span class="p">(</span><span class="n">param_count</span> <span class="o">*</span> <span class="mi">4</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">)</span> <span class="c1"># FP32</span>
|
||
|
||
<span class="c1"># Measure accuracy</span>
|
||
<span class="n">predictions</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
<span class="n">pred_labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">predictions</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">accuracy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">pred_labels</span> <span class="o">==</span> <span class="n">y_test</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Measure latency (statistical rigor with 100 runs)</span>
|
||
<span class="n">latencies</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_runs</span><span class="p">):</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||
<span class="n">_</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">X_test</span><span class="p">[:</span><span class="mi">1</span><span class="p">])</span> <span class="c1"># Single-sample inference</span>
|
||
<span class="n">latencies</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start</span><span class="p">)</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span> <span class="c1"># ms</span>
|
||
|
||
<span class="n">avg_latency</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">latencies</span><span class="p">)</span>
|
||
<span class="n">std_latency</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">latencies</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Store comprehensive metrics</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">metrics</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'parameter_count'</span><span class="p">:</span> <span class="nb">int</span><span class="p">(</span><span class="n">param_count</span><span class="p">),</span>
|
||
<span class="s1">'model_size_mb'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">model_size_mb</span><span class="p">),</span>
|
||
<span class="s1">'accuracy'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">accuracy</span><span class="p">),</span>
|
||
<span class="s1">'latency_ms_mean'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">avg_latency</span><span class="p">),</span>
|
||
<span class="s1">'latency_ms_std'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">std_latency</span><span class="p">),</span>
|
||
<span class="s1">'throughput_samples_per_sec'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="mi">1000</span> <span class="o">/</span> <span class="n">avg_latency</span><span class="p">)</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metrics</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why This Design</strong>: Single class captures all necessary information for reproducible benchmarking. System info ensures results can be contextualized. Multiple latency runs provide statistical confidence.</p>
|
||
</section>
|
||
<section id="submission-generation-function">
|
||
<h4>2. Submission Generation Function<a class="headerlink" href="#submission-generation-function" title="Link to this heading">#</a></h4>
|
||
<p><strong>Standardized JSON Schema Following MLPerf Format</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">generate_submission</span><span class="p">(</span>
|
||
<span class="n">baseline_report</span><span class="p">:</span> <span class="n">BenchmarkReport</span><span class="p">,</span>
|
||
<span class="n">optimized_report</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">BenchmarkReport</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||
<span class="n">student_name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||
<span class="n">techniques_applied</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Generate standardized benchmark submission.</span>
|
||
|
||
<span class="sd"> Creates MLPerf-inspired JSON with:</span>
|
||
<span class="sd"> - Version and timestamp metadata</span>
|
||
<span class="sd"> - System information for reproducibility</span>
|
||
<span class="sd"> - Baseline performance metrics</span>
|
||
<span class="sd"> - Optional optimized metrics with techniques</span>
|
||
<span class="sd"> - Automatic improvement calculation</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> baseline_report: BenchmarkReport for baseline model</span>
|
||
<span class="sd"> optimized_report: Optional BenchmarkReport for optimized version</span>
|
||
<span class="sd"> student_name: Optional submitter name</span>
|
||
<span class="sd"> techniques_applied: List of optimization techniques used</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> Dictionary ready for JSON serialization</span>
|
||
<span class="sd"> """</span>
|
||
<span class="n">submission</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'tinytorch_version'</span><span class="p">:</span> <span class="s1">'0.1.0'</span><span class="p">,</span>
|
||
<span class="s1">'submission_type'</span><span class="p">:</span> <span class="s1">'capstone_benchmark'</span><span class="p">,</span>
|
||
<span class="s1">'timestamp'</span><span class="p">:</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">timestamp</span><span class="p">,</span>
|
||
<span class="s1">'system_info'</span><span class="p">:</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">system_info</span><span class="p">,</span>
|
||
<span class="s1">'baseline'</span><span class="p">:</span> <span class="p">{</span>
|
||
<span class="s1">'model_name'</span><span class="p">:</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">model_name</span><span class="p">,</span>
|
||
<span class="s1">'metrics'</span><span class="p">:</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">metrics</span>
|
||
<span class="p">}</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="c1"># Add optional student name</span>
|
||
<span class="k">if</span> <span class="n">student_name</span><span class="p">:</span>
|
||
<span class="n">submission</span><span class="p">[</span><span class="s1">'student_name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">student_name</span>
|
||
|
||
<span class="c1"># Add optimization results if provided</span>
|
||
<span class="k">if</span> <span class="n">optimized_report</span><span class="p">:</span>
|
||
<span class="n">submission</span><span class="p">[</span><span class="s1">'optimized'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'model_name'</span><span class="p">:</span> <span class="n">optimized_report</span><span class="o">.</span><span class="n">model_name</span><span class="p">,</span>
|
||
<span class="s1">'metrics'</span><span class="p">:</span> <span class="n">optimized_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">,</span>
|
||
<span class="s1">'techniques_applied'</span><span class="p">:</span> <span class="n">techniques_applied</span> <span class="ow">or</span> <span class="p">[]</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="c1"># Automatically calculate improvement metrics</span>
|
||
<span class="n">baseline_lat</span> <span class="o">=</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'latency_ms_mean'</span><span class="p">]</span>
|
||
<span class="n">optimized_lat</span> <span class="o">=</span> <span class="n">optimized_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'latency_ms_mean'</span><span class="p">]</span>
|
||
<span class="n">baseline_size</span> <span class="o">=</span> <span class="n">baseline_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'model_size_mb'</span><span class="p">]</span>
|
||
<span class="n">optimized_size</span> <span class="o">=</span> <span class="n">optimized_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'model_size_mb'</span><span class="p">]</span>
|
||
|
||
<span class="n">submission</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'speedup'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">baseline_lat</span> <span class="o">/</span> <span class="n">optimized_lat</span><span class="p">),</span>
|
||
<span class="s1">'compression_ratio'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">baseline_size</span> <span class="o">/</span> <span class="n">optimized_size</span><span class="p">),</span>
|
||
<span class="s1">'accuracy_delta'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span>
|
||
<span class="n">optimized_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'accuracy'</span><span class="p">]</span> <span class="o">-</span>
|
||
<span class="n">baseline_report</span><span class="o">.</span><span class="n">metrics</span><span class="p">[</span><span class="s1">'accuracy'</span><span class="p">]</span>
|
||
<span class="p">)</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="k">return</span> <span class="n">submission</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">save_submission</span><span class="p">(</span><span class="n">submission</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span> <span class="n">filepath</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Save submission to JSON file with proper formatting."""</span>
|
||
<span class="n">Path</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span><span class="o">.</span><span class="n">write_text</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">submission</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"✅ Submission saved to: </span><span class="si">{</span><span class="n">filepath</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">filepath</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Submission Schema Example</strong>:</p>
|
||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"tinytorch_version"</span><span class="p">:</span><span class="w"> </span><span class="s2">"0.1.0"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"submission_type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"capstone_benchmark"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"timestamp"</span><span class="p">:</span><span class="w"> </span><span class="s2">"2025-01-15 14:23:41"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"system_info"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"platform"</span><span class="p">:</span><span class="w"> </span><span class="s2">"macOS-14.2-arm64"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"python_version"</span><span class="p">:</span><span class="w"> </span><span class="s2">"3.11.6"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"numpy_version"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1.24.3"</span>
|
||
<span class="w"> </span><span class="p">},</span>
|
||
<span class="w"> </span><span class="nt">"baseline"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"model_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"baseline_mlp"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"metrics"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"parameter_count"</span><span class="p">:</span><span class="w"> </span><span class="mi">263</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"model_size_mb"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.001</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"accuracy"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.35</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"latency_ms_mean"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.042</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"latency_ms_std"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.008</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"throughput_samples_per_sec"</span><span class="p">:</span><span class="w"> </span><span class="mf">23809.52</span>
|
||
<span class="w"> </span><span class="p">}</span>
|
||
<span class="w"> </span><span class="p">},</span>
|
||
<span class="w"> </span><span class="nt">"optimized"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"model_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"optimized_mlp"</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"metrics"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"parameter_count"</span><span class="p">:</span><span class="w"> </span><span class="mi">198</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"model_size_mb"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00075</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"accuracy"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.33</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"latency_ms_mean"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.031</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"latency_ms_std"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.006</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"throughput_samples_per_sec"</span><span class="p">:</span><span class="w"> </span><span class="mf">32258.06</span>
|
||
<span class="w"> </span><span class="p">},</span>
|
||
<span class="w"> </span><span class="nt">"techniques_applied"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"pruning"</span><span class="p">,</span><span class="w"> </span><span class="s2">"quantization"</span><span class="p">]</span>
|
||
<span class="w"> </span><span class="p">},</span>
|
||
<span class="w"> </span><span class="nt">"improvements"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nt">"speedup"</span><span class="p">:</span><span class="w"> </span><span class="mf">1.35</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"compression_ratio"</span><span class="p">:</span><span class="w"> </span><span class="mf">1.33</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nt">"accuracy_delta"</span><span class="p">:</span><span class="w"> </span><span class="mf">-0.02</span>
|
||
<span class="w"> </span><span class="p">}</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why This Schema</strong>: MLPerf-inspired format ensures reproducibility. System info enables verification. Normalized metrics (speedup, compression ratio) work across hardware. Automatic improvement calculation prevents manual errors.</p>
|
||
</section>
|
||
<section id="complete-optimization-workflow">
|
||
<h4>3. Complete Optimization Workflow<a class="headerlink" href="#complete-optimization-workflow" title="Link to this heading">#</a></h4>
|
||
<p><strong>Bringing Modules 14-19 Together</strong></p>
|
||
<p>This workflow demonstrates the full optimization pipeline:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">run_optimization_workflow_example</span><span class="p">():</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Complete optimization workflow using Modules 14-19.</span>
|
||
|
||
<span class="sd"> Pipeline:</span>
|
||
<span class="sd"> 1. Profile baseline (Module 14)</span>
|
||
<span class="sd"> 2. Apply optimizations (Modules 15-18)</span>
|
||
<span class="sd"> 3. Benchmark with rigor (Module 19)</span>
|
||
<span class="sd"> 4. Generate submission (Module 20)</span>
|
||
|
||
<span class="sd"> This is the COMPLETE story of TinyTorch optimization!</span>
|
||
<span class="sd"> """</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"="</span><span class="o">*</span><span class="mi">70</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"TINYTORCH OPTIMIZATION WORKFLOW"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"="</span><span class="o">*</span><span class="mi">70</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Import optimization APIs</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.perf.profiling</span><span class="w"> </span><span class="kn">import</span> <span class="n">Profiler</span><span class="p">,</span> <span class="n">quick_profile</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.perf.compression</span><span class="w"> </span><span class="kn">import</span> <span class="n">magnitude_prune</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.benchmarking</span><span class="w"> </span><span class="kn">import</span> <span class="n">Benchmark</span><span class="p">,</span> <span class="n">BenchmarkResult</span>
|
||
|
||
<span class="c1"># Step 1: Profile baseline model (Module 14)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">[STEP 1] Profile Baseline - Module 14"</span><span class="p">)</span>
|
||
<span class="n">baseline_model</span> <span class="o">=</span> <span class="n">SimpleMLP</span><span class="p">(</span><span class="n">input_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">hidden_size</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">profiler</span> <span class="o">=</span> <span class="n">Profiler</span><span class="p">()</span>
|
||
<span class="c1"># Optional: Use Module 14's profiler for detailed analysis</span>
|
||
<span class="c1"># profile_data = quick_profile(baseline_model, input_tensor)</span>
|
||
|
||
<span class="c1"># Step 2: Benchmark baseline (Module 19)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">[STEP 2] Benchmark Baseline - Module 19"</span><span class="p">)</span>
|
||
<span class="n">baseline_report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"baseline_mlp"</span><span class="p">)</span>
|
||
<span class="n">baseline_report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">baseline_model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">50</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Step 3: Apply optimizations (Modules 15-18)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">[STEP 3] Apply Optimizations - Modules 15-18"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">" Available APIs:"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">" - Module 15: quantize_model(model, bits=8)"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">" - Module 16: magnitude_prune(model, sparsity=0.5)"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">" - Module 17: enable_kv_cache(model) # For transformers"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">" - Module 18: Use accelerated ops"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Example: Apply pruning (students can add quantization, etc.)</span>
|
||
<span class="n">optimized_model</span> <span class="o">=</span> <span class="n">baseline_model</span> <span class="c1"># Apply real optimizations here</span>
|
||
<span class="c1"># optimized_model = magnitude_prune(baseline_model, sparsity=0.3)</span>
|
||
<span class="c1"># optimized_model = quantize_model(optimized_model, bits=8)</span>
|
||
|
||
<span class="c1"># Step 4: Benchmark optimized version (Module 19)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">[STEP 4] Benchmark Optimized - Module 19"</span><span class="p">)</span>
|
||
<span class="n">optimized_report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"optimized_mlp"</span><span class="p">)</span>
|
||
<span class="n">optimized_report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">optimized_model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">50</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Step 5: Generate submission (Module 20)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">[STEP 5] Generate Submission - Module 20"</span><span class="p">)</span>
|
||
<span class="n">submission</span> <span class="o">=</span> <span class="n">generate_submission</span><span class="p">(</span>
|
||
<span class="n">baseline_report</span><span class="o">=</span><span class="n">baseline_report</span><span class="p">,</span>
|
||
<span class="n">optimized_report</span><span class="o">=</span><span class="n">optimized_report</span><span class="p">,</span>
|
||
<span class="n">student_name</span><span class="o">=</span><span class="s2">"TinyTorch Optimizer"</span><span class="p">,</span>
|
||
<span class="n">techniques_applied</span><span class="o">=</span><span class="p">[</span><span class="s2">"pruning"</span><span class="p">,</span> <span class="s2">"quantization"</span><span class="p">]</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="c1"># Display improvements</span>
|
||
<span class="k">if</span> <span class="s1">'improvements'</span> <span class="ow">in</span> <span class="n">submission</span><span class="p">:</span>
|
||
<span class="n">imp</span> <span class="o">=</span> <span class="n">submission</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">]</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="se">\n</span><span class="s2"> 📈 Results:"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" Speedup: </span><span class="si">{</span><span class="n">imp</span><span class="p">[</span><span class="s1">'speedup'</span><span class="p">]</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2">x"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" Compression: </span><span class="si">{</span><span class="n">imp</span><span class="p">[</span><span class="s1">'compression_ratio'</span><span class="p">]</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2">x"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" Accuracy Δ: </span><span class="si">{</span><span class="n">imp</span><span class="p">[</span><span class="s1">'accuracy_delta'</span><span class="p">]</span><span class="o">*</span><span class="mi">100</span><span class="si">:</span><span class="s2">+.1f</span><span class="si">}</span><span class="s2">%"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Step 6: Save submission</span>
|
||
<span class="n">save_submission</span><span class="p">(</span><span class="n">submission</span><span class="p">,</span> <span class="s2">"optimization_submission.json"</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">✅ Complete optimization workflow demonstrated!"</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">submission</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why This Workflow Matters</strong>: Shows how TinyTorch modules work together as a cohesive framework. Students see the complete optimization story: measure → optimize → validate → submit. This workflow pattern applies to real production ML perf.</p>
|
||
</section>
|
||
</section>
|
||
<section id="connection-to-tinytorch-optimization-tier">
|
||
<h3>Connection to TinyTorch Optimization Tier<a class="headerlink" href="#connection-to-tinytorch-optimization-tier" title="Link to this heading">#</a></h3>
|
||
<p>This capstone brings together the entire Optimization Tier:</p>
|
||
<p><strong>The Complete Optimization Story</strong>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Module 14 (Profiling)
|
||
↓
|
||
Identify bottlenecks
|
||
↓
|
||
Modules 15-18 (Optimization Techniques)
|
||
↓
|
||
Apply targeted optimizations
|
||
↓
|
||
Module 19 (Benchmarking)
|
||
↓
|
||
Measure improvements with statistics
|
||
↓
|
||
Module 20 (Submission)
|
||
↓
|
||
Package results for sharing
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>How Modules Work Together</strong>:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Module 14 (Profiling)</strong>: Identifies bottlenecks (memory-bound vs compute-bound)</p></li>
|
||
<li><p><strong>Module 15 (Quantization)</strong>: Reduces precision to save memory and improve throughput</p></li>
|
||
<li><p><strong>Module 16 (Compression)</strong>: Prunes parameters to reduce model size</p></li>
|
||
<li><p><strong>Module 17 (Memoization)</strong>: Caches computations to avoid redundant work</p></li>
|
||
<li><p><strong>Module 18 (Acceleration)</strong>: Applies operator fusion and vectorization</p></li>
|
||
<li><p><strong>Module 19 (Benchmarking)</strong>: Validates optimizations with statistical rigor</p></li>
|
||
<li><p><strong>Module 20 (Submission)</strong>: Packages everything into shareable format</p></li>
|
||
</ol>
|
||
<p><strong>Real-World Application</strong>: This workflow mirrors how production ML teams optimize models:</p>
|
||
<ul class="simple">
|
||
<li><p>Google TPU teams profile → optimize → benchmark → deploy</p></li>
|
||
<li><p>OpenAI profiles GPT training → applies gradient checkpointing → validates memory savings</p></li>
|
||
<li><p>Meta benchmarks PyTorch inference → fuses operators → measures latency improvements</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="enabling-future-competitions">
|
||
<h3>Enabling Future Competitions<a class="headerlink" href="#enabling-future-competitions" title="Link to this heading">#</a></h3>
|
||
<p>The submission infrastructure you build enables future TinyTorch challenges:</p>
|
||
<p><strong>Wake Vision Competition (Coming Soon)</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>Optimize computer vision models for edge deployment</p></li>
|
||
<li><p>Constraints: Latency < 100ms, Model size < 5MB, Accuracy > 85%</p></li>
|
||
<li><p>Rankings based on normalized submissions (same format you’re building)</p></li>
|
||
</ul>
|
||
<p><strong>Custom Competitions</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>Educational settings: Classroom competitions using TinyTorch</p></li>
|
||
<li><p>Research benchmarks: Reproducible optimization studies</p></li>
|
||
<li><p>Community challenges: Open-source ML optimization contests</p></li>
|
||
</ul>
|
||
<p><strong>Extensibility</strong>: The submission format you implement can be extended with:</p>
|
||
<ul class="simple">
|
||
<li><p>Additional metrics (energy consumption, memory bandwidth)</p></li>
|
||
<li><p>Constraint validation (checking competition requirements)</p></li>
|
||
<li><p>Leaderboard integration (automated ranking systems)</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="getting-started">
|
||
<h2>Getting Started<a class="headerlink" href="#getting-started" title="Link to this heading">#</a></h2>
|
||
<section id="prerequisites">
|
||
<h3>Prerequisites<a class="headerlink" href="#prerequisites" title="Link to this heading">#</a></h3>
|
||
<p>Ensure you understand benchmarking and optimization techniques:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Activate TinyTorch environment</span>
|
||
<span class="nb">source</span><span class="w"> </span>scripts/activate-tinytorch
|
||
|
||
<span class="c1"># Required: Benchmarking methodology (Module 19)</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>benchmarking
|
||
|
||
<span class="c1"># Helpful: Optimization techniques (Modules 14-18)</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>profiling<span class="w"> </span><span class="c1"># Module 14: Find bottlenecks</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>quantization<span class="w"> </span><span class="c1"># Module 15: Reduce precision</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>compression<span class="w"> </span><span class="c1"># Module 16: Prune parameters</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>memoization<span class="w"> </span><span class="c1"># Module 17: Cache computations</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>acceleration<span class="w"> </span><span class="c1"># Module 18: Operator fusion</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why Module 19 is Essential</strong>: This capstone uses Module 19’s <code class="docutils literal notranslate"><span class="pre">BenchmarkReport</span></code> class as the foundation. Understanding statistical measurement methodology from Module 19 is critical for generating valid submissions.</p>
|
||
</section>
|
||
<section id="development-workflow">
|
||
<h3>Development Workflow<a class="headerlink" href="#development-workflow" title="Link to this heading">#</a></h3>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Open the development file</strong>: <code class="docutils literal notranslate"><span class="pre">modules/20_capstone/20_capstone.py</span></code></p></li>
|
||
<li><p><strong>Implement SimpleMLP</strong>: Simple demonstration model for benchmarking</p></li>
|
||
<li><p><strong>Build BenchmarkReport</strong>: Class to collect and store metrics</p></li>
|
||
<li><p><strong>Create generate_submission()</strong>: Function to create standardized JSON</p></li>
|
||
<li><p><strong>Add save_submission()</strong>: JSON serialization with proper formatting</p></li>
|
||
<li><p><strong>Implement workflow examples</strong>: Basic and optimization workflow demonstrations</p></li>
|
||
<li><p><strong>Export and verify</strong>: <code class="docutils literal notranslate"><span class="pre">tito</span> <span class="pre">module</span> <span class="pre">complete</span> <span class="pre">20</span> <span class="pre">&&</span> <span class="pre">tito</span> <span class="pre">test</span> <span class="pre">capstone</span></code></p></li>
|
||
</ol>
|
||
<p><strong>Development Tips</strong>:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Test BenchmarkReport with toy model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">SimpleMLP</span><span class="p">(</span><span class="n">input_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">hidden_size</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"test_model"</span><span class="p">)</span>
|
||
<span class="n">metrics</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Verify all required metrics are present</span>
|
||
<span class="n">required</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'parameter_count'</span><span class="p">,</span> <span class="s1">'model_size_mb'</span><span class="p">,</span> <span class="s1">'accuracy'</span><span class="p">,</span>
|
||
<span class="s1">'latency_ms_mean'</span><span class="p">,</span> <span class="s1">'latency_ms_std'</span><span class="p">,</span> <span class="s1">'throughput_samples_per_sec'</span><span class="p">]</span>
|
||
<span class="k">assert</span> <span class="nb">all</span><span class="p">(</span><span class="n">metric</span> <span class="ow">in</span> <span class="n">metrics</span> <span class="k">for</span> <span class="n">metric</span> <span class="ow">in</span> <span class="n">required</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test submission generation</span>
|
||
<span class="n">submission</span> <span class="o">=</span> <span class="n">generate_submission</span><span class="p">(</span><span class="n">report</span><span class="p">,</span> <span class="n">student_name</span><span class="o">=</span><span class="s2">"Test"</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="s1">'baseline'</span> <span class="ow">in</span> <span class="n">submission</span>
|
||
<span class="k">assert</span> <span class="s1">'system_info'</span> <span class="ow">in</span> <span class="n">submission</span>
|
||
<span class="k">assert</span> <span class="n">submission</span><span class="p">[</span><span class="s1">'submission_type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'capstone_benchmark'</span>
|
||
|
||
<span class="c1"># Test with optimization comparison</span>
|
||
<span class="n">optimized_report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"optimized"</span><span class="p">)</span>
|
||
<span class="c1"># ... benchmark optimized model ...</span>
|
||
<span class="n">submission_with_opt</span> <span class="o">=</span> <span class="n">generate_submission</span><span class="p">(</span><span class="n">report</span><span class="p">,</span> <span class="n">optimized_report</span><span class="p">,</span>
|
||
<span class="n">techniques_applied</span><span class="o">=</span><span class="p">[</span><span class="s2">"pruning"</span><span class="p">])</span>
|
||
<span class="k">assert</span> <span class="s1">'improvements'</span> <span class="ow">in</span> <span class="n">submission_with_opt</span>
|
||
<span class="k">assert</span> <span class="s1">'speedup'</span> <span class="ow">in</span> <span class="n">submission_with_opt</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="testing">
|
||
<h2>Testing<a class="headerlink" href="#testing" title="Link to this heading">#</a></h2>
|
||
<section id="comprehensive-test-suite">
|
||
<h3>Comprehensive Test Suite<a class="headerlink" href="#comprehensive-test-suite" title="Link to this heading">#</a></h3>
|
||
<p>Run the full test suite to verify submission infrastructure:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># TinyTorch CLI (recommended)</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>capstone
|
||
|
||
<span class="c1"># Direct pytest execution</span>
|
||
python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>tests/<span class="w"> </span>-k<span class="w"> </span>capstone<span class="w"> </span>-v
|
||
|
||
<span class="c1"># Expected output:</span>
|
||
<span class="c1"># ✅ test_simple_mlp - Model creation and forward pass</span>
|
||
<span class="c1"># ✅ test_benchmark_report - Metrics collection and storage</span>
|
||
<span class="c1"># ✅ test_submission_generation - JSON creation</span>
|
||
<span class="c1"># ✅ test_submission_schema - Schema validation</span>
|
||
<span class="c1"># ✅ test_submission_with_optimization - Before/after comparison</span>
|
||
<span class="c1"># ✅ test_improvements_calculation - Speedup/compression/accuracy</span>
|
||
<span class="c1"># ✅ test_json_serialization - File saving and loading</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="test-coverage-areas">
|
||
<h3>Test Coverage Areas<a class="headerlink" href="#test-coverage-areas" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p>✅ <strong>SimpleMLP Model</strong>: Forward pass, parameter counting, output shape validation</p></li>
|
||
<li><p>✅ <strong>BenchmarkReport</strong>: Metric collection, system info capture, statistical measurement</p></li>
|
||
<li><p>✅ <strong>Submission Generation</strong>: Schema structure, field presence, type validation</p></li>
|
||
<li><p>✅ <strong>Schema Validation</strong>: Required fields, value ranges, type correctness</p></li>
|
||
<li><p>✅ <strong>Optimization Comparison</strong>: Improvements calculation, technique tracking</p></li>
|
||
<li><p>✅ <strong>JSON Serialization</strong>: File writing, round-trip preservation, formatting</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="inline-testing-validation">
|
||
<h3>Inline Testing & Validation<a class="headerlink" href="#inline-testing-validation" title="Link to this heading">#</a></h3>
|
||
<p>The module includes comprehensive unit tests:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">SimpleMLP</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Model</span> <span class="n">creation</span> <span class="k">with</span> <span class="n">custom</span> <span class="n">parameters</span>
|
||
<span class="err">✅</span> <span class="n">Parameter</span> <span class="n">count</span><span class="p">:</span> <span class="mi">263</span> <span class="p">(</span><span class="mi">10</span><span class="err">×</span><span class="mi">20</span> <span class="o">+</span> <span class="mi">20</span> <span class="o">+</span> <span class="mi">20</span><span class="err">×</span><span class="mi">3</span> <span class="o">+</span> <span class="mi">3</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Forward</span> <span class="k">pass</span> <span class="n">output</span> <span class="n">shape</span><span class="p">:</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">No</span> <span class="n">NaN</span> <span class="n">values</span> <span class="ow">in</span> <span class="n">output</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">SimpleMLP</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">BenchmarkReport</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Model</span> <span class="n">name</span> <span class="ow">and</span> <span class="n">timestamp</span> <span class="nb">set</span> <span class="n">correctly</span>
|
||
<span class="err">✅</span> <span class="n">System</span> <span class="n">info</span> <span class="n">collected</span> <span class="p">(</span><span class="n">platform</span><span class="p">,</span> <span class="n">python_version</span><span class="p">,</span> <span class="n">numpy_version</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Metrics</span><span class="p">:</span> <span class="n">parameter_count</span><span class="p">,</span> <span class="n">model_size_mb</span><span class="p">,</span> <span class="n">accuracy</span><span class="p">,</span> <span class="n">latency</span><span class="p">,</span> <span class="n">throughput</span>
|
||
<span class="err">✅</span> <span class="n">Metric</span> <span class="n">types</span> <span class="ow">and</span> <span class="n">ranges</span> <span class="n">validated</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">BenchmarkReport</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Submission</span> <span class="n">Generation</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Baseline</span> <span class="n">submission</span> <span class="n">structure</span> <span class="n">complete</span>
|
||
<span class="err">✅</span> <span class="n">Version</span><span class="p">,</span> <span class="nb">type</span><span class="p">,</span> <span class="n">timestamp</span><span class="p">,</span> <span class="n">system_info</span><span class="p">,</span> <span class="n">baseline</span> <span class="n">present</span>
|
||
<span class="err">✅</span> <span class="n">Student</span> <span class="n">name</span> <span class="n">included</span> <span class="n">when</span> <span class="n">provided</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">generate_submission</span><span class="p">()</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Submission</span> <span class="n">Schema</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Required</span> <span class="n">fields</span> <span class="n">present</span>
|
||
<span class="err">✅</span> <span class="n">Field</span> <span class="n">types</span> <span class="n">correct</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Baseline</span> <span class="ow">and</span> <span class="n">metrics</span> <span class="n">structure</span> <span class="n">validated</span>
|
||
<span class="err">✅</span> <span class="n">System</span> <span class="n">info</span> <span class="n">contains</span> <span class="n">platform</span> <span class="ow">and</span> <span class="n">python_version</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Schema</span> <span class="n">validation</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Submission</span> <span class="k">with</span> <span class="n">Optimization</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Optimized</span> <span class="n">section</span> <span class="n">present</span> <span class="k">with</span> <span class="n">techniques</span>
|
||
<span class="err">✅</span> <span class="n">Improvements</span> <span class="n">section</span> <span class="k">with</span> <span class="n">speedup</span><span class="p">,</span> <span class="n">compression</span><span class="p">,</span> <span class="n">accuracy_delta</span>
|
||
<span class="err">✅</span> <span class="n">Techniques</span> <span class="nb">list</span> <span class="n">matches</span> <span class="nb">input</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Optimization</span> <span class="n">comparison</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Improvements</span> <span class="n">Calculation</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Speedup</span><span class="p">:</span> <span class="mf">2.0</span><span class="n">x</span> <span class="p">(</span><span class="n">baseline</span> <span class="mf">10.0</span><span class="n">ms</span> <span class="o">/</span> <span class="n">optimized</span> <span class="mf">5.0</span><span class="n">ms</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Compression</span><span class="p">:</span> <span class="mf">2.0</span><span class="n">x</span> <span class="p">(</span><span class="n">baseline</span> <span class="mf">4.0</span><span class="n">MB</span> <span class="o">/</span> <span class="n">optimized</span> <span class="mf">2.0</span><span class="n">MB</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Accuracy</span> <span class="n">delta</span><span class="p">:</span> <span class="o">-</span><span class="mf">0.05</span> <span class="p">(</span><span class="mf">0.75</span> <span class="o">-</span> <span class="mf">0.80</span><span class="p">)</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Improvements</span> <span class="n">math</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🔬</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">JSON</span> <span class="n">Serialization</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">File</span> <span class="n">created</span> <span class="ow">and</span> <span class="n">exists</span>
|
||
<span class="err">✅</span> <span class="n">JSON</span> <span class="n">valid</span> <span class="ow">and</span> <span class="n">loadable</span>
|
||
<span class="err">✅</span> <span class="n">Structure</span> <span class="n">preserved</span> <span class="p">(</span><span class="n">version</span><span class="p">,</span> <span class="n">student_name</span><span class="p">,</span> <span class="n">metrics</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Round</span><span class="o">-</span><span class="n">trip</span> <span class="n">serialization</span> <span class="n">successful</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">File</span> <span class="n">I</span><span class="o">/</span><span class="n">O</span> <span class="err">✓</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="manual-testing-examples">
|
||
<h3>Manual Testing Examples<a class="headerlink" href="#manual-testing-examples" title="Link to this heading">#</a></h3>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.capstone</span><span class="w"> </span><span class="kn">import</span> <span class="n">SimpleMLP</span><span class="p">,</span> <span class="n">BenchmarkReport</span><span class="p">,</span> <span class="n">generate_submission</span><span class="p">,</span> <span class="n">save_submission</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">np</span>
|
||
|
||
<span class="c1"># Example 1: Basic benchmark workflow</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">42</span><span class="p">)</span>
|
||
<span class="n">X_test</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
<span class="n">y_test</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
|
||
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">SimpleMLP</span><span class="p">(</span><span class="n">input_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">hidden_size</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"simple_mlp"</span><span class="p">)</span>
|
||
<span class="n">metrics</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">50</span><span class="p">)</span>
|
||
|
||
<span class="n">submission</span> <span class="o">=</span> <span class="n">generate_submission</span><span class="p">(</span><span class="n">report</span><span class="p">,</span> <span class="n">student_name</span><span class="o">=</span><span class="s2">"Your Name"</span><span class="p">)</span>
|
||
<span class="n">save_submission</span><span class="p">(</span><span class="n">submission</span><span class="p">,</span> <span class="s2">"my_submission.json"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Example 2: Optimization comparison workflow</span>
|
||
<span class="n">baseline_model</span> <span class="o">=</span> <span class="n">SimpleMLP</span><span class="p">(</span><span class="n">input_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">hidden_size</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">baseline_report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"baseline"</span><span class="p">)</span>
|
||
<span class="n">baseline_report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">baseline_model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">50</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Apply optimizations (example: smaller model)</span>
|
||
<span class="n">optimized_model</span> <span class="o">=</span> <span class="n">SimpleMLP</span><span class="p">(</span><span class="n">input_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">hidden_size</span><span class="o">=</span><span class="mi">15</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">optimized_report</span> <span class="o">=</span> <span class="n">BenchmarkReport</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">"optimized"</span><span class="p">)</span>
|
||
<span class="n">optimized_report</span><span class="o">.</span><span class="n">benchmark_model</span><span class="p">(</span><span class="n">optimized_model</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">num_runs</span><span class="o">=</span><span class="mi">50</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Generate comparison submission</span>
|
||
<span class="n">submission</span> <span class="o">=</span> <span class="n">generate_submission</span><span class="p">(</span>
|
||
<span class="n">baseline_report</span><span class="o">=</span><span class="n">baseline_report</span><span class="p">,</span>
|
||
<span class="n">optimized_report</span><span class="o">=</span><span class="n">optimized_report</span><span class="p">,</span>
|
||
<span class="n">student_name</span><span class="o">=</span><span class="s2">"Optimizer"</span><span class="p">,</span>
|
||
<span class="n">techniques_applied</span><span class="o">=</span><span class="p">[</span><span class="s2">"architecture_search"</span><span class="p">,</span> <span class="s2">"pruning"</span><span class="p">]</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Speedup: </span><span class="si">{</span><span class="n">submission</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">][</span><span class="s1">'speedup'</span><span class="p">]</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2">x"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Compression: </span><span class="si">{</span><span class="n">submission</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">][</span><span class="s1">'compression_ratio'</span><span class="p">]</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2">x"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Accuracy Δ: </span><span class="si">{</span><span class="n">submission</span><span class="p">[</span><span class="s1">'improvements'</span><span class="p">][</span><span class="s1">'accuracy_delta'</span><span class="p">]</span><span class="o">*</span><span class="mi">100</span><span class="si">:</span><span class="s2">+.1f</span><span class="si">}</span><span class="s2">%"</span><span class="p">)</span>
|
||
|
||
<span class="n">save_submission</span><span class="p">(</span><span class="n">submission</span><span class="p">,</span> <span class="s2">"optimization_comparison.json"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="systems-thinking-questions">
|
||
<h2>Systems Thinking Questions<a class="headerlink" href="#systems-thinking-questions" title="Link to this heading">#</a></h2>
|
||
<section id="optimization-workflow-integration">
|
||
<h3>Optimization Workflow Integration<a class="headerlink" href="#optimization-workflow-integration" title="Link to this heading">#</a></h3>
|
||
<p><strong>Question 1: Optimization Interaction</strong></p>
|
||
<p>You apply INT8 quantization (4× memory reduction) followed by 75% magnitude pruning (4× parameter reduction). Should you expect 16× total memory reduction?</p>
|
||
<p><strong>Reflection Structure</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>Quantization affects: Precision per parameter (FP32 → INT8 = 4 bytes → 1 byte)</p></li>
|
||
<li><p>Pruning affects: Parameter count (75% zeroed out)</p></li>
|
||
<li><p>Combined effect: Depends on sparse storage format</p></li>
|
||
<li><p>Why not multiplicative: Dense storage still allocates space for zeros</p></li>
|
||
</ul>
|
||
<p><strong>Systems Insight</strong>: Quantization reduces bits per parameter. Pruning zeros out weights but doesn’t automatically reduce memory in dense format. For true 16× reduction, you need sparse storage (CSR/COO format) that doesn’t allocate space for zeros. This is why Module 16 teaches both pruning AND sparse representations.</p>
|
||
</section>
|
||
<section id="submission-reproducibility">
|
||
<h3>Submission Reproducibility<a class="headerlink" href="#submission-reproducibility" title="Link to this heading">#</a></h3>
|
||
<p><strong>Question 2: System Information Requirements</strong></p>
|
||
<p>Why does the submission schema require <code class="docutils literal notranslate"><span class="pre">system_info</span></code> with platform, Python version, and NumPy version? What breaks if this is omitted?</p>
|
||
<p><strong>Systems Insight</strong>: Reproducibility requires environment specification. NumPy 1.24 vs 2.0 can produce different results due to algorithm changes. Platform affects performance (ARM vs x86, SIMD instruction sets). Python version impacts library behavior. Without system info, results aren’t verifiable—claims of “2× speedup” are meaningless if hardware isn’t specified. Production ML teams learned this the hard way when “optimizations” only worked on specific configurations.</p>
|
||
</section>
|
||
<section id="statistical-measurement-validity">
|
||
<h3>Statistical Measurement Validity<a class="headerlink" href="#statistical-measurement-validity" title="Link to this heading">#</a></h3>
|
||
<p><strong>Question 3: Measurement Rigor</strong></p>
|
||
<p>Your optimized model shows 5% latency improvement with standard deviation of 8%. Is this a real improvement or measurement noise?</p>
|
||
<p><strong>Reflection Points</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>Mean improvement: 5% faster</p></li>
|
||
<li><p>Standard deviation: 8% of baseline latency</p></li>
|
||
<li><p>Confidence interval: Likely overlapping</p></li>
|
||
<li><p>Statistical significance: Requires hypothesis testing</p></li>
|
||
</ul>
|
||
<p><strong>Systems Insight</strong>: When std > improvement magnitude, difference could be noise. Proper approach: run t-test with p < 0.05 threshold. Module 19’s benchmarking teaches this—multiple runs + confidence intervals prevent false claims. Production teams don’t deploy “optimizations” without statistical confidence because regressions cost money.</p>
|
||
</section>
|
||
<section id="workflow-scalability">
|
||
<h3>Workflow Scalability<a class="headerlink" href="#workflow-scalability" title="Link to this heading">#</a></h3>
|
||
<p><strong>Question 4: Production Scaling</strong></p>
|
||
<p>How does this submission workflow scale to production models with millions of parameters and hours-long training runs?</p>
|
||
<p><strong>Reflection</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>SimpleMLP benchmarks in milliseconds → GPT-2 trains for days</p></li>
|
||
<li><p>Toy dataset (100 samples) → Production (billions of tokens)</p></li>
|
||
<li><p>Single metric focus → Multi-objective optimization (latency + memory + throughput + cost)</p></li>
|
||
</ul>
|
||
<p><strong>Systems Insight</strong>: The workflow patterns are identical—profile, optimize, benchmark, submit—but tools must scale. Production uses distributed profiling (across GPUs/nodes), long-running benchmarks (days not minutes), and comprehensive metrics (MLPerf includes 20+ metrics). TinyTorch teaches the workflow; PyTorch provides production-scale infrastructure.</p>
|
||
</section>
|
||
<section id="competition-fairness">
|
||
<h3>Competition Fairness<a class="headerlink" href="#competition-fairness" title="Link to this heading">#</a></h3>
|
||
<p><strong>Question 5: Normalized Metrics Design</strong></p>
|
||
<p>Why use speedup ratios (baseline_time / optimized_time) instead of absolute times (10ms → 5ms) for competition ranking?</p>
|
||
<p><strong>Systems Insight</strong>: Hardware variability makes absolute times meaningless for comparison. M1 Mac vs Intel i9 vs AMD Threadripper all produce different absolute times. But 2× speedup is meaningful across hardware—same relative improvement. Speedup ratios enable fair comparison and focus on optimization quality, not hardware access. MLPerf competitions use normalized metrics for exactly this reason.</p>
|
||
</section>
|
||
</section>
|
||
<section id="ready-to-complete-the-optimization-tier">
|
||
<h2>Ready to Complete the Optimization Tier?<a class="headerlink" href="#ready-to-complete-the-optimization-tier" title="Link to this heading">#</a></h2>
|
||
<p>You’ve reached the capstone of TinyTorch’s Optimization Tier. This submission infrastructure brings together everything from Modules 14-19, transforming individual optimization techniques into a cohesive workflow that mirrors production ML engineering.</p>
|
||
<p><strong>What You’ll Achieve</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p>Complete optimization workflow: Profile → Optimize → Benchmark → Submit</p></li>
|
||
<li><p>Professional submission infrastructure enabling future competitions</p></li>
|
||
<li><p>Understanding of how TinyTorch modules work together as a framework</p></li>
|
||
<li><p>Reproducible, shareable results demonstrating your optimization skills</p></li>
|
||
</ul>
|
||
<p><strong>The Capstone Mindset</strong>:</p>
|
||
<blockquote>
|
||
<div><p>“Individual modules teach techniques. The capstone teaches workflow. Production ML isn’t about knowing tools—it’s about orchestrating them effectively.”
|
||
— Every ML systems engineer</p>
|
||
</div></blockquote>
|
||
<p><strong>What’s Next</strong>:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Milestone 05</strong>: Build TinyGPT using your complete framework</p></li>
|
||
<li><p><strong>Wake Vision Competition</strong>: Apply optimization skills to real challenges</p></li>
|
||
<li><p><strong>Community Sharing</strong>: Submit your results, compare with others</p></li>
|
||
</ul>
|
||
<p>This capstone demonstrates you don’t just understand optimization techniques—you can apply them systematically to produce measurable, reproducible improvements. That’s the difference between knowing tools and being an ML systems engineer.</p>
|
||
<p>Choose your preferred way to engage with this capstone:</p>
|
||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||
<div class="sd-row sd-row-cols-1 sd-row-cols-xs-1 sd-row-cols-sm-2 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
🚀 Launch Binder</div>
|
||
<p class="sd-card-text">Run this capstone interactively in your browser. No installation required!</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://mybinder.org/v2/gh/mlsysbook/TinyTorch/main?filepath=modules/20_capstone/20_capstone.py"><span>https://mybinder.org/v2/gh/mlsysbook/TinyTorch/main?filepath=modules/20_capstone/20_capstone.py</span></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
⚡ Open in Colab</div>
|
||
<p class="sd-card-text">Use Google Colab for cloud compute power and easy sharing.</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://colab.research.google.com/github/mlsysbook/TinyTorch/blob/main/modules/20_capstone/20_capstone.ipynb"><span>https://colab.research.google.com/github/mlsysbook/TinyTorch/blob/main/modules/20_capstone/20_capstone.ipynb</span></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
📖 View Source</div>
|
||
<p class="sd-card-text">Browse the Python source code and understand the implementation.</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://github.com/mlsysbook/TinyTorch/blob/main/modules/20_capstone/20_capstone.py"><span>https://github.com/mlsysbook/TinyTorch/blob/main/modules/20_capstone/20_capstone.py</span></a></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">💡 Local Development Recommended</p>
|
||
<p>This capstone involves benchmarking workflows that benefit from consistent hardware and persistent results. Local setup provides better control over measurement conditions and faster iteration cycles.</p>
|
||
<p><strong>Setup</strong>: <code class="docutils literal notranslate"><span class="pre">git</span> <span class="pre">clone</span> <span class="pre">https://github.com/mlsysbook/TinyTorch.git</span> <span class="pre">&&</span> <span class="pre">source</span> <span class="pre">scripts/activate-tinytorch</span> <span class="pre">&&</span> <span class="pre">cd</span> <span class="pre">modules/20_capstone</span></code></p>
|
||
</div>
|
||
<hr class="docutils" />
|
||
<div class="prev-next-area">
|
||
<a class="left-prev" href="../19_benchmarking/ABOUT.html" title="previous page">← Module 19: Benchmarking</a>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./modules"
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="../tiers/olympics.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">🏅 Torch Olympics (Module 20)</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="../chapters/00-introduction.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Course Introduction: ML Systems Engineering Through Implementation</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#core-infrastructure-components">Core Infrastructure Components</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#benchmarkreport-class">1. BenchmarkReport Class</a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#submission-generation-function">2. Submission Generation Function</a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-optimization-workflow">3. Complete Optimization Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#connection-to-tinytorch-optimization-tier">Connection to TinyTorch Optimization Tier</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#enabling-future-competitions">Enabling Future Competitions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#optimization-workflow-integration">Optimization Workflow Integration</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#submission-reproducibility">Submission Reproducibility</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#statistical-measurement-validity">Statistical Measurement Validity</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#workflow-scalability">Workflow Scalability</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#competition-fairness">Competition Fairness</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-complete-the-optimization-tier">Ready to Complete the Optimization Tier?</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Prof. Vijay Janapa Reddi (Harvard University)
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2025.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |