mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-05-24 10:05:50 -05:00
1393 lines
123 KiB
HTML
1393 lines
123 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="../" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>01. Tensor — Tiny🔥Torch</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="../_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="../_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/custom.css?v=009d37f4" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="../_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="../_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="../_static/copybutton.js?v=f281be69"></script>
|
||
<script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="../_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'modules/01_tensor_ABOUT';</script>
|
||
<script src="../_static/ml-timeline.js?v=76e9b3e3"></script>
|
||
<script src="../_static/wip-banner.js?v=04a7e74d"></script>
|
||
<script src="../_static/marimo-badges.js?v=e6289128"></script>
|
||
<script src="../_static/sidebar-link.js?v=404b701b"></script>
|
||
<script src="../_static/hero-carousel.js?v=10341d2a"></script>
|
||
<script src="../_static/subscribe-modal.js?v=42919b64"></script>
|
||
<link rel="icon" href="../_static/favicon.svg"/>
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
<link rel="next" title="02. Activations" href="02_activations_ABOUT.html" />
|
||
<link rel="prev" title="🏗 Foundation Tier (Modules 01-07)" href="../tiers/foundation.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="../search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search..."
|
||
aria-label="Search..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../_static/logo-tinytorch.png" class="logo__image only-light" alt="Tiny🔥Torch - Home"/>
|
||
<script>document.write(`<img src="../_static/logo-tinytorch.png" class="logo__image only-dark" alt="Tiny🔥Torch - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🚀 Getting Started</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../getting-started.html">Complete Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏗 Foundation Tier (01-07)</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/foundation.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">01. Tensor</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="02_activations_ABOUT.html">02. Activations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="03_layers_ABOUT.html">03. Layers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="04_losses_ABOUT.html">04. Losses</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="05_autograd_ABOUT.html">05. Autograd</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="06_optimizers_ABOUT.html">06. Optimizers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="07_training_ABOUT.html">07. Training</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏛️ Architecture Tier (08-13)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/architecture.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="08_dataloader_ABOUT.html">08. DataLoader</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="09_spatial_ABOUT.html">09. Convolutions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="10_tokenization_ABOUT.html">10. Tokenization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="11_embeddings_ABOUT.html">11. Embeddings</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="12_attention_ABOUT.html">12. Attention</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="13_transformers_ABOUT.html">13. Transformers</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">⏱️ Optimization Tier (14-19)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/optimization.html">📖 Tier Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="14_profiling_ABOUT.html">14. Profiling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="15_quantization_ABOUT.html">15. Quantization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="16_compression_ABOUT.html">16. Compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="17_memoization_ABOUT.html">17. Memoization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="18_acceleration_ABOUT.html">18. Acceleration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="19_benchmarking_ABOUT.html">19. Benchmarking</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏅 Capstone Competition</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tiers/olympics.html">📖 Competition Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="20_capstone_ABOUT.html">20. Torch Olympics</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🧭 Course Orientation</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/00-introduction.html">Course Structure</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../prerequisites.html">Prerequisites & Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/learning-journey.html">Learning Journey</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/milestones.html">Historical Milestones</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🛠️ TITO CLI Reference</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/overview.html">Command Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/modules.html">Module Workflow</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/milestones.html">Milestone System</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/data.html">Progress & Data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito/troubleshooting.html">Troubleshooting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../datasets.html">Datasets Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🤝 Community</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../community.html">Ecosystem</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources.html">Learning Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../credits.html">Credits & Acknowledgments</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="../_sources/modules/01_tensor_ABOUT.md" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.md</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>01. Tensor</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-youll-build">What You’ll Build</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#core-concepts">Core Concepts</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensors-as-multidimensional-arrays">Tensors as Multidimensional Arrays</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#broadcasting-efficient-shape-alignment">Broadcasting: Efficient Shape Alignment</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#views-vs-copies-memory-efficiency">Views vs. Copies: Memory Efficiency</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#computational-complexity">Computational Complexity</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#architecture-overview">Architecture Overview</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensor-class-design">Tensor Class Design</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#data-flow-architecture">Data Flow Architecture</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#module-integration">Module Integration</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensor-class-foundation">Tensor Class Foundation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#arithmetic-operations">Arithmetic Operations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#matrix-multiplication">Matrix Multiplication</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#shape-manipulation">Shape Manipulation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reduction-operations">Reduction Operations</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#production-context">Production Context</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#your-implementation-vs-production-frameworks">Your Implementation vs. Production Frameworks</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#side-by-side-code-comparison">Side-by-Side Code Comparison</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-production-usage">Real-World Production Usage</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics-at-scale">Performance Characteristics at Scale</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#package-integration">Package Integration</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-your-implementation-maps-to-pytorch">How Your Implementation Maps to PyTorch</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#common-pitfalls">Common Pitfalls</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#shape-mismatch-errors">Shape Mismatch Errors</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#broadcasting-confusion">Broadcasting Confusion</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#view-vs-copy-confusion">View vs Copy Confusion</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#axis-parameter-mistakes">Axis Parameter Mistakes</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dtype-issues">Dtype Issues</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-leaks-with-large-tensors">Memory Leaks with Large Tensors</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-applications">Real-World Applications</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-foundations">Mathematical Foundations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics">Performance Characteristics</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#whats-next">What’s Next</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-build">Ready to Build?</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section id="tensor">
|
||
<h1>01. Tensor<a class="headerlink" href="#tensor" title="Link to this heading">#</a></h1>
|
||
<p><strong>FOUNDATION TIER</strong> | Difficulty: ⭐ (1/4) | Time: 4-6 hours</p>
|
||
<section id="overview">
|
||
<h2>Overview<a class="headerlink" href="#overview" title="Link to this heading">#</a></h2>
|
||
<p>The Tensor class is the foundational data structure of machine learning - every neural network, from simple linear models to GPT and Stable Diffusion, operates on tensors. You’ll build N-dimensional arrays from scratch with arithmetic operations, broadcasting, and shape manipulation. This module gives you deep insight into how PyTorch and TensorFlow work under the hood, understanding the memory and performance implications that matter in production ML systems.</p>
|
||
</section>
|
||
<section id="learning-objectives">
|
||
<h2>Learning Objectives<a class="headerlink" href="#learning-objectives" title="Link to this heading">#</a></h2>
|
||
<p>By the end of this module, you will be able to:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Understand memory and performance implications</strong>: Recognize how tensor operations dominate compute time and memory usage in ML systems - a single matrix multiplication can consume 90% of forward pass time in production frameworks like PyTorch</p></li>
|
||
<li><p><strong>Implement core tensor functionality</strong>: Build a complete Tensor class with arithmetic (<code class="docutils literal notranslate"><span class="pre">+</span></code>, <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">*</span></code>, <code class="docutils literal notranslate"><span class="pre">/</span></code>), matrix multiplication, shape manipulation (<code class="docutils literal notranslate"><span class="pre">reshape</span></code>, <code class="docutils literal notranslate"><span class="pre">transpose</span></code>), and reductions (<code class="docutils literal notranslate"><span class="pre">sum</span></code>, <code class="docutils literal notranslate"><span class="pre">mean</span></code>, <code class="docutils literal notranslate"><span class="pre">max</span></code>) with proper error handling and validation</p></li>
|
||
<li><p><strong>Master broadcasting semantics</strong>: Understand NumPy broadcasting rules that enable efficient computations across different tensor shapes without data copying - critical for batch processing and efficient neural network operations</p></li>
|
||
<li><p><strong>Connect to production frameworks</strong>: See how your implementation mirrors PyTorch’s <code class="docutils literal notranslate"><span class="pre">torch.Tensor</span></code> and TensorFlow’s <code class="docutils literal notranslate"><span class="pre">tf.Tensor</span></code> design patterns, understanding the architectural decisions that power real ML systems</p></li>
|
||
<li><p><strong>Analyze performance trade-offs</strong>: Understand computational complexity (O(n³) for matrix multiplication), memory usage patterns (contiguous vs. strided), and when to copy data vs. create views for optimization</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="build-use-reflect">
|
||
<h2>Build → Use → Reflect<a class="headerlink" href="#build-use-reflect" title="Link to this heading">#</a></h2>
|
||
<p>This module follows TinyTorch’s <strong>Build → Use → Reflect</strong> framework:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Build</strong>: Implement the Tensor class from scratch using NumPy as the underlying array library - creating <code class="docutils literal notranslate"><span class="pre">__init__</span></code>, operator overloading (<code class="docutils literal notranslate"><span class="pre">__add__</span></code>, <code class="docutils literal notranslate"><span class="pre">__mul__</span></code>, etc.), shape manipulation methods, and reduction operations</p></li>
|
||
<li><p><strong>Use</strong>: Apply your Tensor to real problems like matrix multiplication for neural network layers, data normalization with broadcasting, and statistical computations across various shapes and dimensions</p></li>
|
||
<li><p><strong>Reflect</strong>: Understand systems-level implications - why tensor operations dominate training time, how memory layout (row-major vs. column-major) affects cache performance, and how broadcasting eliminates redundant data copying</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="what-youll-build">
|
||
<h2>What You’ll Build<a class="headerlink" href="#what-youll-build" title="Link to this heading">#</a></h2>
|
||
<p>By completing this module, you’ll create a production-ready Tensor class with:</p>
|
||
<p><strong>Core Data Structure:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>N-dimensional array wrapper around NumPy with clean API</p></li>
|
||
<li><p>Properties for shape, size, dtype, and data access</p></li>
|
||
<li><p>Dormant gradient tracking attributes (activated in Module 05)</p></li>
|
||
</ul>
|
||
<p><strong>Arithmetic Operations:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Element-wise operations: <code class="docutils literal notranslate"><span class="pre">+</span></code>, <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">*</span></code>, <code class="docutils literal notranslate"><span class="pre">/</span></code>, <code class="docutils literal notranslate"><span class="pre">**</span></code></p></li>
|
||
<li><p>Full broadcasting support for Tensor-Tensor and Tensor-scalar operations</p></li>
|
||
<li><p>Automatic shape alignment following NumPy broadcasting rules</p></li>
|
||
</ul>
|
||
<p><strong>Matrix Operations:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">matmul()</span></code> for matrix multiplication with shape validation</p></li>
|
||
<li><p>Support for matrix-matrix, matrix-vector multiplication</p></li>
|
||
<li><p>Clear error messages for dimension mismatches</p></li>
|
||
</ul>
|
||
<p><strong>Shape Manipulation:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">reshape()</span></code> with -1 inference for automatic dimension calculation</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">transpose()</span></code> for dimension swapping</p></li>
|
||
<li><p>View vs. copy semantics understanding</p></li>
|
||
</ul>
|
||
<p><strong>Reduction Operations:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">sum()</span></code>, <code class="docutils literal notranslate"><span class="pre">mean()</span></code>, <code class="docutils literal notranslate"><span class="pre">max()</span></code>, <code class="docutils literal notranslate"><span class="pre">min()</span></code> with axis parameter</p></li>
|
||
<li><p>Global reductions (entire tensor) and axis-specific reductions</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">keepdims</span></code> support for maintaining dimensionality</p></li>
|
||
</ul>
|
||
<p><strong>Real-World Usage Pattern:</strong>
|
||
Your Tensor enables the fundamental neural network forward pass: <code class="docutils literal notranslate"><span class="pre">output</span> <span class="pre">=</span> <span class="pre">x.matmul(W)</span> <span class="pre">+</span> <span class="pre">b</span></code> - exactly how PyTorch and TensorFlow work internally.</p>
|
||
</section>
|
||
<section id="core-concepts">
|
||
<h2>Core Concepts<a class="headerlink" href="#core-concepts" title="Link to this heading">#</a></h2>
|
||
<section id="tensors-as-multidimensional-arrays">
|
||
<h3>Tensors as Multidimensional Arrays<a class="headerlink" href="#tensors-as-multidimensional-arrays" title="Link to this heading">#</a></h3>
|
||
<p>A tensor is a generalization of scalars (0D), vectors (1D), and matrices (2D) to N dimensions:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Scalar</strong>: <code class="docutils literal notranslate"><span class="pre">Tensor(5.0)</span></code> - shape <code class="docutils literal notranslate"><span class="pre">()</span></code></p></li>
|
||
<li><p><strong>Vector</strong>: <code class="docutils literal notranslate"><span class="pre">Tensor([1,</span> <span class="pre">2,</span> <span class="pre">3])</span></code> - shape <code class="docutils literal notranslate"><span class="pre">(3,)</span></code></p></li>
|
||
<li><p><strong>Matrix</strong>: <code class="docutils literal notranslate"><span class="pre">Tensor([[1,</span> <span class="pre">2],</span> <span class="pre">[3,</span> <span class="pre">4]])</span></code> - shape <code class="docutils literal notranslate"><span class="pre">(2,</span> <span class="pre">2)</span></code></p></li>
|
||
<li><p><strong>3D Tensor</strong>: Image batch <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">height,</span> <span class="pre">width)</span></code> - shape <code class="docutils literal notranslate"><span class="pre">(32,</span> <span class="pre">224,</span> <span class="pre">224)</span></code></p></li>
|
||
<li><p><strong>4D Tensor</strong>: CNN features <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">channels,</span> <span class="pre">height,</span> <span class="pre">width)</span></code> - shape <code class="docutils literal notranslate"><span class="pre">(32,</span> <span class="pre">3,</span> <span class="pre">224,</span> <span class="pre">224)</span></code></p></li>
|
||
</ul>
|
||
<p><strong>Why tensors matter</strong>: They provide a unified interface for all ML data - images, text embeddings, audio spectrograms, and model parameters are all tensors with different shapes.</p>
|
||
</section>
|
||
<section id="broadcasting-efficient-shape-alignment">
|
||
<h3>Broadcasting: Efficient Shape Alignment<a class="headerlink" href="#broadcasting-efficient-shape-alignment" title="Link to this heading">#</a></h3>
|
||
<p>Broadcasting automatically expands smaller tensors to match larger ones without copying data:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Matrix (2,2) + Vector (2,) → broadcasts to (2,2)</span>
|
||
<span class="n">matrix</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span>
|
||
<span class="n">vector</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">])</span>
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">matrix</span> <span class="o">+</span> <span class="n">vector</span> <span class="c1"># [[11, 22], [13, 24]]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Broadcasting rules</strong> (NumPy-compatible):</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Align shapes from right to left</p></li>
|
||
<li><p>Dimensions are compatible if they’re equal or one is 1</p></li>
|
||
<li><p>Missing dimensions are treated as size 1</p></li>
|
||
</ol>
|
||
<p><strong>Why broadcasting matters</strong>: Eliminates redundant data copying. Adding a bias vector to 1000 feature maps broadcasts once instead of copying the vector 1000 times - saving memory and enabling vectorization.</p>
|
||
</section>
|
||
<section id="views-vs-copies-memory-efficiency">
|
||
<h3>Views vs. Copies: Memory Efficiency<a class="headerlink" href="#views-vs-copies-memory-efficiency" title="Link to this heading">#</a></h3>
|
||
<p>Some operations return <strong>views</strong> (sharing memory) vs. <strong>copies</strong> (duplicating data):</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Views</strong> (O(1)): <code class="docutils literal notranslate"><span class="pre">reshape()</span></code>, <code class="docutils literal notranslate"><span class="pre">transpose()</span></code> when possible - no data movement</p></li>
|
||
<li><p><strong>Copies</strong> (O(n)): Arithmetic operations, explicit <code class="docutils literal notranslate"><span class="pre">.copy()</span></code> - duplicate storage</p></li>
|
||
</ul>
|
||
<p><strong>Why this matters</strong>: A view of a 1GB tensor is free (just metadata). A copy allocates another 1GB. Understanding view semantics prevents memory blowup in production systems.</p>
|
||
</section>
|
||
<section id="computational-complexity">
|
||
<h3>Computational Complexity<a class="headerlink" href="#computational-complexity" title="Link to this heading">#</a></h3>
|
||
<p>Different operations have vastly different costs:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Element-wise</strong> (<code class="docutils literal notranslate"><span class="pre">+</span></code>, <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">*</span></code>): O(n) - linear in tensor size</p></li>
|
||
<li><p><strong>Reductions</strong> (<code class="docutils literal notranslate"><span class="pre">sum</span></code>, <code class="docutils literal notranslate"><span class="pre">mean</span></code>): O(n) - must visit every element</p></li>
|
||
<li><p><strong>Matrix multiply</strong> (<code class="docutils literal notranslate"><span class="pre">matmul</span></code>): O(n³) for square matrices - dominates training time</p></li>
|
||
</ul>
|
||
<p><strong>Why this matters</strong>: In a neural network forward pass, matrix multiplications consume 90%+ of compute time. Optimizing matmul is critical - hence specialized hardware (GPUs, TPUs) and libraries (cuBLAS, MKL).</p>
|
||
</section>
|
||
</section>
|
||
<section id="architecture-overview">
|
||
<h2>Architecture Overview<a class="headerlink" href="#architecture-overview" title="Link to this heading">#</a></h2>
|
||
<section id="tensor-class-design">
|
||
<h3>Tensor Class Design<a class="headerlink" href="#tensor-class-design" title="Link to this heading">#</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>┌─────────────────────────────────────────┐
|
||
│ Tensor Class │
|
||
├─────────────────────────────────────────┤
|
||
│ Properties: │
|
||
│ - data: np.ndarray (underlying storage)│
|
||
│ - shape: tuple (dimensions) │
|
||
│ - size: int (total elements) │
|
||
│ - dtype: np.dtype (data type) │
|
||
│ - requires_grad: bool (autograd flag) │
|
||
│ - grad: Tensor (gradient - Module 05) │
|
||
├─────────────────────────────────────────┤
|
||
│ Operator Overloading: │
|
||
│ - __add__, __sub__, __mul__, __truediv__│
|
||
│ - __pow__ (exponentiation) │
|
||
│ - Returns new Tensor instances │
|
||
├─────────────────────────────────────────┤
|
||
│ Methods: │
|
||
│ - matmul(other): Matrix multiplication │
|
||
│ - reshape(*shape): Shape manipulation │
|
||
│ - transpose(): Dimension swap │
|
||
│ - sum/mean/max/min(axis): Reductions │
|
||
└─────────────────────────────────────────┘
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="data-flow-architecture">
|
||
<h3>Data Flow Architecture<a class="headerlink" href="#data-flow-architecture" title="Link to this heading">#</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Python Interface (your code)
|
||
↓
|
||
Tensor Class
|
||
↓
|
||
NumPy Backend (vectorized operations)
|
||
↓
|
||
C/Fortran Libraries (BLAS, LAPACK)
|
||
↓
|
||
Hardware (CPU SIMD, cache)
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Your implementation</strong>: Python wrapper → NumPy
|
||
<strong>PyTorch/TensorFlow</strong>: Python wrapper → C++ engine → GPU kernels</p>
|
||
<p>The architecture is identical in concept - you’re learning the same design patterns used in production, just with NumPy instead of custom CUDA kernels.</p>
|
||
</section>
|
||
<section id="module-integration">
|
||
<h3>Module Integration<a class="headerlink" href="#module-integration" title="Link to this heading">#</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Module 01: Tensor (THIS MODULE)
|
||
↓ provides foundation
|
||
Module 02: Activations (ReLU, Sigmoid operate on Tensors)
|
||
↓ uses tensors
|
||
Module 03: Layers (Linear, Conv2d store weights as Tensors)
|
||
↓ uses tensors
|
||
Module 05: Autograd (adds .grad attribute to Tensors)
|
||
↓ enhances tensors
|
||
Module 06: Optimizers (updates Tensor parameters)
|
||
</pre></div>
|
||
</div>
|
||
<p>Your Tensor is the universal foundation - every subsequent module builds on what you create here.</p>
|
||
</section>
|
||
</section>
|
||
<section id="prerequisites">
|
||
<h2>Prerequisites<a class="headerlink" href="#prerequisites" title="Link to this heading">#</a></h2>
|
||
<p>This is the first module - no prerequisites! Verify your environment is ready:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Activate TinyTorch environment</span>
|
||
<span class="nb">source</span><span class="w"> </span>scripts/activate-tinytorch
|
||
|
||
<span class="c1"># Check system health</span>
|
||
tito<span class="w"> </span>system<span class="w"> </span>health
|
||
</pre></div>
|
||
</div>
|
||
<p>All checks should pass (Python 3.8+, NumPy, pytest installed) before starting.</p>
|
||
</section>
|
||
<section id="getting-started">
|
||
<h2>Getting Started<a class="headerlink" href="#getting-started" title="Link to this heading">#</a></h2>
|
||
<section id="development-workflow">
|
||
<h3>Development Workflow<a class="headerlink" href="#development-workflow" title="Link to this heading">#</a></h3>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Open the development notebook</strong>: <code class="docutils literal notranslate"><span class="pre">modules/01_tensor/tensor_dev.ipynb</span></code> in Jupyter or your preferred editor</p></li>
|
||
<li><p><strong>Implement Tensor.<strong>init</strong></strong>: Create constructor that converts data to NumPy array, stores shape/size/dtype, initializes gradient attributes</p></li>
|
||
<li><p><strong>Build arithmetic operations</strong>: Implement <code class="docutils literal notranslate"><span class="pre">__add__</span></code>, <code class="docutils literal notranslate"><span class="pre">__sub__</span></code>, <code class="docutils literal notranslate"><span class="pre">__mul__</span></code>, <code class="docutils literal notranslate"><span class="pre">__truediv__</span></code> with broadcasting support for both Tensor-Tensor and Tensor-scalar operations</p></li>
|
||
<li><p><strong>Add matrix multiplication</strong>: Implement <code class="docutils literal notranslate"><span class="pre">matmul()</span></code> with shape validation and clear error messages for dimension mismatches</p></li>
|
||
<li><p><strong>Create shape manipulation</strong>: Implement <code class="docutils literal notranslate"><span class="pre">reshape()</span></code> (with -1 support) and <code class="docutils literal notranslate"><span class="pre">transpose()</span></code> for dimension swapping</p></li>
|
||
<li><p><strong>Implement reductions</strong>: Build <code class="docutils literal notranslate"><span class="pre">sum()</span></code>, <code class="docutils literal notranslate"><span class="pre">mean()</span></code>, <code class="docutils literal notranslate"><span class="pre">max()</span></code> with axis parameter and keepdims support</p></li>
|
||
<li><p><strong>Export and verify</strong>: Run <code class="docutils literal notranslate"><span class="pre">tito</span> <span class="pre">export</span> <span class="pre">01</span></code> to export to package, then <code class="docutils literal notranslate"><span class="pre">tito</span> <span class="pre">test</span> <span class="pre">01</span></code> to validate all tests pass</p></li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="implementation-guide">
|
||
<h2>Implementation Guide<a class="headerlink" href="#implementation-guide" title="Link to this heading">#</a></h2>
|
||
<section id="tensor-class-foundation">
|
||
<h3>Tensor Class Foundation<a class="headerlink" href="#tensor-class-foundation" title="Link to this heading">#</a></h3>
|
||
<p>Your Tensor class wraps NumPy arrays and provides ML-specific functionality:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span>
|
||
|
||
<span class="c1"># Create tensors from Python lists or NumPy arrays</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">3.0</span><span class="p">,</span> <span class="mf">4.0</span><span class="p">]])</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">1.5</span><span class="p">],</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">]])</span>
|
||
|
||
<span class="c1"># Properties provide clean API access</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="c1"># (2, 2)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">size</span><span class="p">)</span> <span class="c1"># 4</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span> <span class="c1"># float32</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Implementation details</strong>: You’ll implement <code class="docutils literal notranslate"><span class="pre">__init__</span></code> to convert input data to NumPy arrays, store shape/size/dtype as properties, and initialize dormant gradient attributes (<code class="docutils literal notranslate"><span class="pre">requires_grad</span></code>, <code class="docutils literal notranslate"><span class="pre">grad</span></code>) that activate in Module 05.</p>
|
||
</section>
|
||
<section id="arithmetic-operations">
|
||
<h3>Arithmetic Operations<a class="headerlink" href="#arithmetic-operations" title="Link to this heading">#</a></h3>
|
||
<p>Implement operator overloading for element-wise operations with broadcasting:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Element-wise operations via operator overloading</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span> <span class="c1"># Addition: [[1.5, 3.5], [5.5, 7.5]]</span>
|
||
<span class="n">w</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="n">y</span> <span class="c1"># Element-wise multiplication</span>
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="mi">2</span> <span class="c1"># Exponentiation</span>
|
||
<span class="n">s</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">y</span> <span class="c1"># Subtraction</span>
|
||
<span class="n">d</span> <span class="o">=</span> <span class="n">x</span> <span class="o">/</span> <span class="n">y</span> <span class="c1"># Division</span>
|
||
|
||
<span class="c1"># Broadcasting: scalar operations automatically expand</span>
|
||
<span class="n">scaled</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span> <span class="c1"># [[2.0, 4.0], [6.0, 8.0]]</span>
|
||
<span class="n">shifted</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">10</span> <span class="c1"># [[11.0, 12.0], [13.0, 14.0]]</span>
|
||
|
||
<span class="c1"># Broadcasting: vector + matrix</span>
|
||
<span class="n">matrix</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span>
|
||
<span class="n">vector</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">])</span>
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">matrix</span> <span class="o">+</span> <span class="n">vector</span> <span class="c1"># [[11, 22], [13, 24]]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Systems insight</strong>: These operations vectorize automatically via NumPy, achieving ~100x speedup over Python loops. This is why all ML frameworks use tensors - the performance difference between <code class="docutils literal notranslate"><span class="pre">for</span> <span class="pre">i</span> <span class="pre">in</span> <span class="pre">range(n):</span> <span class="pre">result[i]</span> <span class="pre">=</span> <span class="pre">a[i]</span> <span class="pre">+</span> <span class="pre">b[i]</span></code> and <code class="docutils literal notranslate"><span class="pre">result</span> <span class="pre">=</span> <span class="pre">a</span> <span class="pre">+</span> <span class="pre">b</span></code> is dramatic at scale.</p>
|
||
</section>
|
||
<section id="matrix-multiplication">
|
||
<h3>Matrix Multiplication<a class="headerlink" href="#matrix-multiplication" title="Link to this heading">#</a></h3>
|
||
<p>Matrix multiplication is the heart of neural networks - every layer performs it:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Matrix multiplication (the @ operator)</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span> <span class="c1"># 2×2</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">]])</span> <span class="c1"># 2×2</span>
|
||
<span class="n">c</span> <span class="o">=</span> <span class="n">a</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">b</span><span class="p">)</span> <span class="c1"># 2×2 result: [[19, 22], [43, 50]]</span>
|
||
|
||
<span class="c1"># Neural network forward pass pattern: y = xW + b</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]])</span> <span class="c1"># Input: (batch=2, features=3)</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">]])</span> <span class="c1"># Weights: (3, 2)</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">])</span> <span class="c1"># Bias: (2,)</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">W</span><span class="p">)</span> <span class="o">+</span> <span class="n">b</span> <span class="c1"># (2, 2)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Computational complexity</strong>: For matrices <code class="docutils literal notranslate"><span class="pre">(M,K)</span> <span class="pre">@</span> <span class="pre">(K,N)</span></code>, the cost is <code class="docutils literal notranslate"><span class="pre">O(M×K×N)</span></code> floating-point operations. A 1000×1000 matrix multiplication requires 2 billion FLOPs - this dominates training time in production systems.</p>
|
||
</section>
|
||
<section id="shape-manipulation">
|
||
<h3>Shape Manipulation<a class="headerlink" href="#shape-manipulation" title="Link to this heading">#</a></h3>
|
||
<p>Neural networks constantly reshape tensors to match layer requirements:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Reshape: change interpretation of same data (O(1) operation)</span>
|
||
<span class="n">tensor</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">])</span>
|
||
<span class="n">reshaped</span> <span class="o">=</span> <span class="n">tensor</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span> <span class="c1"># [[1, 2, 3], [4, 5, 6]]</span>
|
||
<span class="n">flat</span> <span class="o">=</span> <span class="n">reshaped</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># [1, 2, 3, 4, 5, 6]</span>
|
||
|
||
<span class="c1"># Transpose: swap dimensions (data rearrangement)</span>
|
||
<span class="n">matrix</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]])</span> <span class="c1"># (2, 3)</span>
|
||
<span class="n">transposed</span> <span class="o">=</span> <span class="n">matrix</span><span class="o">.</span><span class="n">transpose</span><span class="p">()</span> <span class="c1"># (3, 2): [[1, 4], [2, 5], [3, 6]]</span>
|
||
|
||
<span class="c1"># CNN data flow example</span>
|
||
<span class="n">images</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">32</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">224</span><span class="p">,</span> <span class="mi">224</span><span class="p">))</span> <span class="c1"># (batch, channels, H, W)</span>
|
||
<span class="n">features</span> <span class="o">=</span> <span class="n">images</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">32</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># (batch, 3*224*224) - flatten for MLP</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Memory consideration</strong>: <code class="docutils literal notranslate"><span class="pre">reshape</span></code> often returns <em>views</em> (no data copying) when possible - an O(1) operation. <code class="docutils literal notranslate"><span class="pre">transpose</span></code> may require data rearrangement depending on memory layout. Understanding views vs. copies is crucial: views share memory (efficient), copies duplicate data (expensive for large tensors).</p>
|
||
</section>
|
||
<section id="reduction-operations">
|
||
<h3>Reduction Operations<a class="headerlink" href="#reduction-operations" title="Link to this heading">#</a></h3>
|
||
<p>Aggregation operations collapse dimensions for statistics and loss computation:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Reduce along different axes</span>
|
||
<span class="n">total</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="c1"># Scalar: sum all elements</span>
|
||
<span class="n">col_sums</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># Sum columns: [4, 6]</span>
|
||
<span class="n">row_sums</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># Sum rows: [3, 7]</span>
|
||
|
||
<span class="c1"># Statistical reductions</span>
|
||
<span class="n">means</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># Column-wise mean</span>
|
||
<span class="n">minimums</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># Row-wise minimum</span>
|
||
<span class="n">maximums</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="c1"># Global maximum</span>
|
||
|
||
<span class="c1"># Batch loss averaging (common pattern)</span>
|
||
<span class="n">losses</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">])</span> <span class="c1"># Per-sample losses</span>
|
||
<span class="n">avg_loss</span> <span class="o">=</span> <span class="n">losses</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="c1"># 0.45 - batch average</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Production pattern</strong>: Every loss function uses reductions. Cross-entropy loss computes per-sample losses then averages: <code class="docutils literal notranslate"><span class="pre">loss</span> <span class="pre">=</span> <span class="pre">-log(predictions[correct_class]).mean()</span></code>. Understanding axis semantics prevents bugs in multi-dimensional operations.</p>
|
||
</section>
|
||
</section>
|
||
<section id="testing">
|
||
<h2>Testing<a class="headerlink" href="#testing" title="Link to this heading">#</a></h2>
|
||
<section id="comprehensive-test-suite">
|
||
<h3>Comprehensive Test Suite<a class="headerlink" href="#comprehensive-test-suite" title="Link to this heading">#</a></h3>
|
||
<p>Run the full test suite to verify tensor functionality:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># TinyTorch CLI (recommended - runs all 01_tensor tests)</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="m">01</span>
|
||
|
||
<span class="c1"># Direct pytest execution (more verbose output)</span>
|
||
python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>tests/01_tensor/<span class="w"> </span>-v
|
||
|
||
<span class="c1"># Run specific test class</span>
|
||
python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>tests/01_tensor/test_tensor_core.py::TestTensorCreation<span class="w"> </span>-v
|
||
</pre></div>
|
||
</div>
|
||
<p>Expected output: All tests pass with green checkmarks showing your Tensor implementation works correctly.</p>
|
||
</section>
|
||
<section id="test-coverage-areas">
|
||
<h3>Test Coverage Areas<a class="headerlink" href="#test-coverage-areas" title="Link to this heading">#</a></h3>
|
||
<p>Your implementation is validated across these dimensions:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Initialization</strong> (<code class="docutils literal notranslate"><span class="pre">test_tensor_from_list</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_from_numpy</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_shapes</span></code>): Creating tensors from Python lists, NumPy arrays, and nested structures with correct shape/dtype handling</p></li>
|
||
<li><p><strong>Arithmetic Operations</strong> (<code class="docutils literal notranslate"><span class="pre">test_tensor_addition</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_multiplication</span></code>): Element-wise addition, subtraction, multiplication, division with both Tensor-Tensor and Tensor-scalar combinations</p></li>
|
||
<li><p><strong>Broadcasting</strong> (<code class="docutils literal notranslate"><span class="pre">test_scalar_broadcasting</span></code>, <code class="docutils literal notranslate"><span class="pre">test_vector_broadcasting</span></code>): Automatic shape alignment for different tensor shapes, scalar expansion, matrix-vector broadcasting</p></li>
|
||
<li><p><strong>Matrix Multiplication</strong> (<code class="docutils literal notranslate"><span class="pre">test_matrix_multiplication</span></code>): Matrix-matrix, matrix-vector multiplication with shape validation and error handling for incompatible dimensions</p></li>
|
||
<li><p><strong>Shape Manipulation</strong> (<code class="docutils literal notranslate"><span class="pre">test_tensor_reshape</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_transpose</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_flatten</span></code>): Reshape with -1 inference, transpose with dimension swapping, validation for incompatible sizes</p></li>
|
||
<li><p><strong>Reductions</strong> (<code class="docutils literal notranslate"><span class="pre">test_sum</span></code>, <code class="docutils literal notranslate"><span class="pre">test_mean</span></code>, <code class="docutils literal notranslate"><span class="pre">test_max</span></code>): Aggregation along various axes (None, 0, 1, multiple), keepdims behavior, global vs. axis-specific reduction</p></li>
|
||
<li><p><strong>Memory Management</strong> (<code class="docutils literal notranslate"><span class="pre">test_tensor_data_access</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_copy_semantics</span></code>, <code class="docutils literal notranslate"><span class="pre">test_tensor_memory_efficiency</span></code>): Data access patterns, copy vs. view semantics, memory usage validation</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="inline-testing-validation">
|
||
<h3>Inline Testing & Validation<a class="headerlink" href="#inline-testing-validation" title="Link to this heading">#</a></h3>
|
||
<p>The development notebook includes comprehensive inline tests with immediate feedback:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Example inline test output</span>
|
||
<span class="err">🧪</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Tensor</span> <span class="n">Creation</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Tensor</span> <span class="n">created</span> <span class="kn">from</span><span class="w"> </span><span class="nn">list</span>
|
||
<span class="err">✅</span> <span class="n">Shape</span> <span class="nb">property</span> <span class="n">correct</span><span class="p">:</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="err">✅</span> <span class="n">Size</span> <span class="nb">property</span> <span class="n">correct</span><span class="p">:</span> <span class="mi">4</span>
|
||
<span class="err">✅</span> <span class="n">dtype</span> <span class="ow">is</span> <span class="n">float32</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Tensor</span> <span class="n">initialization</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🧪</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Arithmetic</span> <span class="n">Operations</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="n">Addition</span><span class="p">:</span> <span class="p">[[</span><span class="mi">6</span><span class="p">,</span> <span class="mi">8</span><span class="p">],</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">12</span><span class="p">]]</span>
|
||
<span class="err">✅</span> <span class="n">Multiplication</span> <span class="n">works</span> <span class="n">element</span><span class="o">-</span><span class="n">wise</span>
|
||
<span class="err">✅</span> <span class="n">Broadcasting</span><span class="p">:</span> <span class="n">scalar</span> <span class="o">+</span> <span class="n">tensor</span>
|
||
<span class="err">✅</span> <span class="n">Broadcasting</span><span class="p">:</span> <span class="n">matrix</span> <span class="o">+</span> <span class="n">vector</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Arithmetic</span> <span class="n">operations</span> <span class="err">✓</span>
|
||
|
||
<span class="err">🧪</span> <span class="n">Unit</span> <span class="n">Test</span><span class="p">:</span> <span class="n">Matrix</span> <span class="n">Multiplication</span><span class="o">...</span>
|
||
<span class="err">✅</span> <span class="mi">2</span><span class="err">×</span><span class="mi">2</span> <span class="o">@</span> <span class="mi">2</span><span class="err">×</span><span class="mi">2</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">19</span><span class="p">,</span> <span class="mi">22</span><span class="p">],</span> <span class="p">[</span><span class="mi">43</span><span class="p">,</span> <span class="mi">50</span><span class="p">]]</span>
|
||
<span class="err">✅</span> <span class="n">Shape</span> <span class="n">validation</span> <span class="n">catches</span> <span class="mi">2</span><span class="err">×</span><span class="mi">2</span> <span class="o">@</span> <span class="mi">3</span><span class="err">×</span><span class="mi">1</span> <span class="n">error</span>
|
||
<span class="err">✅</span> <span class="n">Error</span> <span class="n">message</span> <span class="n">shows</span><span class="p">:</span> <span class="s2">"2 ≠ 3"</span>
|
||
<span class="err">📈</span> <span class="n">Progress</span><span class="p">:</span> <span class="n">Matrix</span> <span class="n">operations</span> <span class="err">✓</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="manual-testing-examples">
|
||
<h3>Manual Testing Examples<a class="headerlink" href="#manual-testing-examples" title="Link to this heading">#</a></h3>
|
||
<p>Validate your implementation interactively:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">np</span>
|
||
|
||
<span class="c1"># Test basic operations</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">]])</span>
|
||
|
||
<span class="k">assert</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="p">(</span><span class="n">x</span> <span class="o">+</span> <span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="o">==</span> <span class="p">[[</span><span class="mi">6</span><span class="p">,</span> <span class="mi">8</span><span class="p">],</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">12</span><span class="p">]]</span>
|
||
<span class="k">assert</span> <span class="n">x</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">data</span> <span class="o">==</span> <span class="mi">10</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"✓ Basic operations working"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test broadcasting</span>
|
||
<span class="n">small</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="n">small</span>
|
||
<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="o">==</span> <span class="p">[[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">6</span><span class="p">]]</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"✓ Broadcasting functional"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test reductions</span>
|
||
<span class="n">col_means</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="n">np</span><span class="o">.</span><span class="n">allclose</span><span class="p">(</span><span class="n">col_means</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="p">[</span><span class="mf">2.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"✓ Reductions working"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test neural network pattern: y = xW + b</span>
|
||
<span class="n">batch</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]])</span> <span class="c1"># (2, 3)</span>
|
||
<span class="n">weights</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">]])</span> <span class="c1"># (3, 2)</span>
|
||
<span class="n">bias</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">])</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="o">+</span> <span class="n">bias</span>
|
||
<span class="k">assert</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"✓ Neural network forward pass pattern works!"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="production-context">
|
||
<h2>Production Context<a class="headerlink" href="#production-context" title="Link to this heading">#</a></h2>
|
||
<section id="your-implementation-vs-production-frameworks">
|
||
<h3>Your Implementation vs. Production Frameworks<a class="headerlink" href="#your-implementation-vs-production-frameworks" title="Link to this heading">#</a></h3>
|
||
<p>Understanding what you’re building vs. what production frameworks provide:</p>
|
||
<div class="pst-scrollable-table-container"><table class="table">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Feature</p></th>
|
||
<th class="head"><p>Your Tensor (Module 01)</p></th>
|
||
<th class="head"><p>PyTorch torch.Tensor</p></th>
|
||
<th class="head"><p>TensorFlow tf.Tensor</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><strong>Backend</strong></p></td>
|
||
<td><p>NumPy (CPU-only)</p></td>
|
||
<td><p>C++/CUDA (CPU/GPU/TPU)</p></td>
|
||
<td><p>C++/CUDA/XLA</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><strong>Dtype Support</strong></p></td>
|
||
<td><p>float32 (primary)</p></td>
|
||
<td><p>float16/32/64, int8/16/32/64, bool, complex</p></td>
|
||
<td><p>Same + bfloat16</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><strong>Operations</strong></p></td>
|
||
<td><p>Arithmetic, matmul, reshape, transpose, reductions</p></td>
|
||
<td><p>1000+ operations</p></td>
|
||
<td><p>1000+ operations</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><strong>Broadcasting</strong></p></td>
|
||
<td><p>✅ Full NumPy rules</p></td>
|
||
<td><p>✅ Same rules</p></td>
|
||
<td><p>✅ Same rules</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><strong>Autograd</strong></p></td>
|
||
<td><p>Dormant (activates Module 05)</p></td>
|
||
<td><p>✅ Full computation graph</p></td>
|
||
<td><p>✅ GradientTape</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><strong>GPU Support</strong></p></td>
|
||
<td><p>❌ CPU-only</p></td>
|
||
<td><p>✅ CUDA, Metal, ROCm</p></td>
|
||
<td><p>✅ CUDA, TPU</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><strong>Memory Pooling</strong></p></td>
|
||
<td><p>❌ Python GC</p></td>
|
||
<td><p>✅ Caching allocator</p></td>
|
||
<td><p>✅ Memory pools</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><strong>JIT Compilation</strong></p></td>
|
||
<td><p>❌ Interpreted</p></td>
|
||
<td><p>✅ TorchScript, torch.compile</p></td>
|
||
<td><p>✅ XLA, TF Graph</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><strong>Distributed</strong></p></td>
|
||
<td><p>❌ Single process</p></td>
|
||
<td><p>✅ DDP, FSDP</p></td>
|
||
<td><p>✅ tf.distribute</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<p><strong>Educational focus</strong>: Your implementation prioritizes clarity and understanding over performance. The core concepts (broadcasting, shape manipulation, reductions) are identical - you’re learning the same patterns used in production, just with simpler infrastructure.</p>
|
||
<p><strong>Line count</strong>: Your implementation is ~1927 lines in the notebook (including tests and documentation). PyTorch’s tensor implementation spans 50,000+ lines across multiple C++ files - your simplified version captures the essential concepts.</p>
|
||
</section>
|
||
<section id="side-by-side-code-comparison">
|
||
<h3>Side-by-Side Code Comparison<a class="headerlink" href="#side-by-side-code-comparison" title="Link to this heading">#</a></h3>
|
||
<p><strong>Your implementation:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span>
|
||
|
||
<span class="c1"># Create tensors</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span>
|
||
<span class="n">w</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">]])</span>
|
||
|
||
<span class="c1"># Forward pass</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">w</span><span class="p">)</span> <span class="c1"># (2,2) @ (2,2) → (2,2)</span>
|
||
<span class="n">loss</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="c1"># Scalar loss</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Equivalent PyTorch (production):</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">torch</span>
|
||
|
||
<span class="c1"># Create tensors (GPU-enabled)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
|
||
<span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Forward pass (automatic gradient tracking)</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">x</span> <span class="o">@</span> <span class="n">w</span> <span class="c1"># Uses cuBLAS for GPU acceleration</span>
|
||
<span class="n">loss</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="c1"># Builds computation graph for backprop</span>
|
||
<span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span> <span class="c1"># Automatic differentiation</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Key differences:</strong></p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>GPU Support</strong>: PyTorch tensors can move to GPU (<code class="docutils literal notranslate"><span class="pre">.cuda()</span></code>) for 10-100x speedup via parallel processing</p></li>
|
||
<li><p><strong>Autograd</strong>: PyTorch automatically tracks operations and computes gradients - you’ll build this in Module 05</p></li>
|
||
<li><p><strong>Memory Pooling</strong>: PyTorch reuses GPU memory via caching allocator - avoids expensive malloc/free calls</p></li>
|
||
<li><p><strong>Optimized Kernels</strong>: PyTorch uses cuBLAS/cuDNN (GPU) and Intel MKL (CPU) - hand-tuned assembly for max performance</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="real-world-production-usage">
|
||
<h3>Real-World Production Usage<a class="headerlink" href="#real-world-production-usage" title="Link to this heading">#</a></h3>
|
||
<p><strong>Meta (Facebook AI)</strong>: PyTorch was developed at Meta and powers their recommendation systems, computer vision models, and LLaMA language models. Their production infrastructure processes billions of tensor operations per second.</p>
|
||
<p><strong>Tesla</strong>: Uses PyTorch tensors for Autopilot neural networks. Each camera frame (6-9 cameras) is converted to tensors, processed through vision models (millions of parameters stored as tensors), and outputs driving decisions in real-time at 36 FPS.</p>
|
||
<p><strong>OpenAI</strong>: GPT-4 training involved tensors with billions of parameters distributed across thousands of GPUs. Each training step performs matrix multiplications on tensors larger than single GPU memory.</p>
|
||
<p><strong>Google</strong>: TensorFlow powers Google Search, Translate, Photos, and Assistant. Google’s TPUs (Tensor Processing Units) are custom hardware designed specifically for accelerating tensor operations.</p>
|
||
</section>
|
||
<section id="performance-characteristics-at-scale">
|
||
<h3>Performance Characteristics at Scale<a class="headerlink" href="#performance-characteristics-at-scale" title="Link to this heading">#</a></h3>
|
||
<p><strong>Memory usage</strong>: GPT-3 scale models (175B parameters) require ~350GB memory just for weights stored as float16 tensors (175B × 2 bytes). Mixed precision training (float16/float32) reduces memory by 2x while maintaining accuracy.</p>
|
||
<p><strong>Computational bottlenecks</strong>: In production training, tensor operations consume 95%+ of runtime. A single linear layer’s matrix multiplication might take 100ms of a 110ms forward pass - optimizing tensor operations is critical.</p>
|
||
<p><strong>Cache efficiency</strong>: Modern CPUs have ~32KB L1 cache, ~256KB L2, ~8MB L3. Accessing memory in tensor-friendly patterns (contiguous, row-major) can be 10-100x faster than cache-unfriendly patterns (strided, column-major).</p>
|
||
</section>
|
||
<section id="package-integration">
|
||
<h3>Package Integration<a class="headerlink" href="#package-integration" title="Link to this heading">#</a></h3>
|
||
<p>After export, your Tensor implementation becomes the foundation of TinyTorch:</p>
|
||
<p><strong>Package Export</strong>: Code exports to <code class="docutils literal notranslate"><span class="pre">tinytorch.core.tensor</span></code></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># When students install tinytorch, they import YOUR work:</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span> <span class="c1"># Your implementation!</span>
|
||
|
||
<span class="c1"># Future modules build on YOUR tensor:</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.activations</span><span class="w"> </span><span class="kn">import</span> <span class="n">ReLU</span> <span class="c1"># Module 02 - operates on your Tensors</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.layers</span><span class="w"> </span><span class="kn">import</span> <span class="n">Linear</span> <span class="c1"># Module 03 - uses your Tensor for weights</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.autograd</span><span class="w"> </span><span class="kn">import</span> <span class="n">backward</span> <span class="c1"># Module 05 - adds gradients to your Tensor</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.optimizers</span><span class="w"> </span><span class="kn">import</span> <span class="n">SGD</span> <span class="c1"># Module 06 - updates your Tensor parameters</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Package structure:</strong></p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>tinytorch/
|
||
├── core/
|
||
│ ├── tensor.py ← YOUR implementation exports here
|
||
│ ├── activations.py ← Module 02 builds on your Tensor
|
||
│ ├── layers.py ← Module 03 builds on your Tensor
|
||
│ ├── losses.py ← Module 04 builds on your Tensor
|
||
│ ├── autograd.py ← Module 05 adds gradients to your Tensor
|
||
│ ├── optimizers.py ← Module 06 updates your Tensor weights
|
||
│ └── ...
|
||
</pre></div>
|
||
</div>
|
||
<p>Your Tensor class is the universal foundation - every subsequent module depends on what you build here.</p>
|
||
</section>
|
||
<section id="how-your-implementation-maps-to-pytorch">
|
||
<h3>How Your Implementation Maps to PyTorch<a class="headerlink" href="#how-your-implementation-maps-to-pytorch" title="Link to this heading">#</a></h3>
|
||
<p><strong>What you just built:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Your TinyTorch Tensor implementation</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.core.tensor</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tensor</span>
|
||
|
||
<span class="c1"># Create a tensor</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]])</span>
|
||
|
||
<span class="c1"># Core operations you implemented</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">2</span> <span class="c1"># Broadcasting</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="c1"># Matrix multiplication</span>
|
||
<span class="n">mean</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># Reductions</span>
|
||
<span class="n">reshaped</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># Shape manipulation</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>How PyTorch does it:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># PyTorch equivalent</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">torch</span>
|
||
|
||
<span class="c1"># Create a tensor</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Same operations, identical semantics</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">2</span> <span class="c1"># Broadcasting (same rules)</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">x</span> <span class="o">@</span> <span class="n">other</span> <span class="c1"># Matrix multiplication (@ operator)</span>
|
||
<span class="n">mean</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># Reductions (dim instead of axis)</span>
|
||
<span class="n">reshaped</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># Shape manipulation (same API)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Key Insight</strong>: Your implementation uses the <strong>same mathematical operations and design patterns</strong> that PyTorch uses internally. The <code class="docutils literal notranslate"><span class="pre">@</span></code> operator is syntactic sugar for matrix multiplication—the actual computation is identical. Broadcasting rules, shape semantics, and reduction operations all follow the same NumPy conventions.</p>
|
||
<p><strong>What’s the SAME?</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Tensor abstraction and API design</p></li>
|
||
<li><p>Broadcasting rules and memory layout principles</p></li>
|
||
<li><p>Shape manipulation semantics (<code class="docutils literal notranslate"><span class="pre">reshape</span></code>, <code class="docutils literal notranslate"><span class="pre">transpose</span></code>)</p></li>
|
||
<li><p>Reduction operation behavior (<code class="docutils literal notranslate"><span class="pre">sum</span></code>, <code class="docutils literal notranslate"><span class="pre">mean</span></code>, <code class="docutils literal notranslate"><span class="pre">max</span></code>)</p></li>
|
||
<li><p>Conceptual architecture: data + operations + metadata</p></li>
|
||
</ul>
|
||
<p><strong>What’s different in production PyTorch?</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Backend</strong>: C++/CUDA for 10-100× speed vs. NumPy</p></li>
|
||
<li><p><strong>GPU support</strong>: <code class="docutils literal notranslate"><span class="pre">.cuda()</span></code> moves tensors to GPU for parallel processing</p></li>
|
||
<li><p><strong>Autograd integration</strong>: <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code> enables automatic differentiation (you’ll build this in Module 05)</p></li>
|
||
<li><p><strong>Memory optimization</strong>: Caching allocator reuses GPU memory, avoiding expensive malloc/free</p></li>
|
||
</ul>
|
||
<p><strong>Why this matters</strong>: When you debug PyTorch code, you’ll understand what’s happening under tensor operations because you implemented them yourself. Shape mismatch errors, broadcasting bugs, memory issues—you know exactly how they work internally, not just how to call the API.</p>
|
||
<p><strong>Production usage example</strong>:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># PyTorch production code (after TinyTorch)</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">torch.nn</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">nn</span>
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">MLPLayer</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">):</span>
|
||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">)</span> <span class="c1"># Uses torch.Tensor internally</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="c1"># Matrix multiply + bias (same as your Tensor.matmul)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>After building your own Tensor class, you understand that <code class="docutils literal notranslate"><span class="pre">nn.Linear(in_features,</span> <span class="pre">out_features)</span></code> is essentially creating weight and bias tensors, then performing <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">@</span> <span class="pre">weights</span> <span class="pre">+</span> <span class="pre">bias</span></code> with your same broadcasting and matmul operations—just optimized in C++/CUDA.</p>
|
||
</section>
|
||
</section>
|
||
<section id="common-pitfalls">
|
||
<h2>Common Pitfalls<a class="headerlink" href="#common-pitfalls" title="Link to this heading">#</a></h2>
|
||
<section id="shape-mismatch-errors">
|
||
<h3>Shape Mismatch Errors<a class="headerlink" href="#shape-mismatch-errors" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: Matrix multiplication fails with cryptic errors like “shapes (2,3) and (2,2) not aligned”</p>
|
||
<p><strong>Solution</strong>: Always verify inner dimensions match: <code class="docutils literal notranslate"><span class="pre">(M,K)</span> <span class="pre">@</span> <span class="pre">(K,N)</span></code> requires K to be equal. Add shape validation with clear error messages:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">a</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">b</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
|
||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot multiply (</span><span class="si">{</span><span class="n">a</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">,</span><span class="si">{</span><span class="n">a</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2">) @ (</span><span class="si">{</span><span class="n">b</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">,</span><span class="si">{</span><span class="n">b</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">a</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2"> ≠ </span><span class="si">{</span><span class="n">b</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="broadcasting-confusion">
|
||
<h3>Broadcasting Confusion<a class="headerlink" href="#broadcasting-confusion" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: Expected <code class="docutils literal notranslate"><span class="pre">(2,3)</span> <span class="pre">+</span> <span class="pre">(3,)</span></code> to broadcast but got error</p>
|
||
<p><strong>Solution</strong>: Broadcasting aligns shapes <em>from the right</em>. <code class="docutils literal notranslate"><span class="pre">(2,3)</span> <span class="pre">+</span> <span class="pre">(3,)</span></code> works (broadcasts to <code class="docutils literal notranslate"><span class="pre">(2,3)</span></code>), but <code class="docutils literal notranslate"><span class="pre">(2,3)</span> <span class="pre">+</span> <span class="pre">(2,)</span></code> fails. Add dimension with reshape if needed: <code class="docutils literal notranslate"><span class="pre">tensor.reshape(2,1)</span></code> to make <code class="docutils literal notranslate"><span class="pre">(2,1)</span></code> broadcastable with <code class="docutils literal notranslate"><span class="pre">(2,3)</span></code>.</p>
|
||
</section>
|
||
<section id="view-vs-copy-confusion">
|
||
<h3>View vs Copy Confusion<a class="headerlink" href="#view-vs-copy-confusion" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: Modified a reshaped tensor and original changed unexpectedly</p>
|
||
<p><strong>Solution</strong>: <code class="docutils literal notranslate"><span class="pre">reshape()</span></code> returns a <em>view</em> when possible - they share memory. Changes to the view affect the original. Use <code class="docutils literal notranslate"><span class="pre">.copy()</span></code> if you need independent data:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">view</span> <span class="o">=</span> <span class="n">tensor</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span> <span class="c1"># Shares memory</span>
|
||
<span class="n">copy</span> <span class="o">=</span> <span class="n">tensor</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="c1"># Independent storage</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="axis-parameter-mistakes">
|
||
<h3>Axis Parameter Mistakes<a class="headerlink" href="#axis-parameter-mistakes" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: <code class="docutils literal notranslate"><span class="pre">sum(axis=1)</span></code> on <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">features)</span></code> returned wrong shape</p>
|
||
<p><strong>Solution</strong>: Axis semantics: <code class="docutils literal notranslate"><span class="pre">axis=0</span></code> reduces over first dimension (batch), <code class="docutils literal notranslate"><span class="pre">axis=1</span></code> reduces over second (features). For <code class="docutils literal notranslate"><span class="pre">(32,</span> <span class="pre">128)</span></code> tensor, <code class="docutils literal notranslate"><span class="pre">sum(axis=0)</span></code> gives <code class="docutils literal notranslate"><span class="pre">(128,)</span></code>, <code class="docutils literal notranslate"><span class="pre">sum(axis=1)</span></code> gives <code class="docutils literal notranslate"><span class="pre">(32,)</span></code>. Visualize which dimension you’re collapsing.</p>
|
||
</section>
|
||
<section id="dtype-issues">
|
||
<h3>Dtype Issues<a class="headerlink" href="#dtype-issues" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: Lost precision after operations, or got integer division instead of float</p>
|
||
<p><strong>Solution</strong>: NumPy defaults to preserving dtype. Integer tensors do integer division (<code class="docutils literal notranslate"><span class="pre">5</span> <span class="pre">/</span> <span class="pre">2</span> <span class="pre">=</span> <span class="pre">2</span></code>). Always create tensors with float dtype explicitly: <code class="docutils literal notranslate"><span class="pre">Tensor([[1,</span> <span class="pre">2]],</span> <span class="pre">dtype=np.float32)</span></code> or convert: <code class="docutils literal notranslate"><span class="pre">tensor.astype(np.float32)</span></code>.</p>
|
||
</section>
|
||
<section id="memory-leaks-with-large-tensors">
|
||
<h3>Memory Leaks with Large Tensors<a class="headerlink" href="#memory-leaks-with-large-tensors" title="Link to this heading">#</a></h3>
|
||
<p><strong>Problem</strong>: Memory usage grows unbounded during training loop</p>
|
||
<p><strong>Solution</strong>: Clear intermediate results in loops. Don’t accumulate tensors in lists unnecessarily. Use in-place operations when safe. Example:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Bad: accumulates memory</span>
|
||
<span class="n">losses</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
||
<span class="n">loss</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
|
||
<span class="n">losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span> <span class="c1"># Keeps all tensors in memory</span>
|
||
|
||
<span class="c1"># Good: extract values</span>
|
||
<span class="n">losses</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
||
<span class="n">loss</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
|
||
<span class="n">losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">loss</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">item</span><span class="p">())</span> <span class="c1"># Store scalar, release tensor</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="systems-thinking-questions">
|
||
<h2>Systems Thinking Questions<a class="headerlink" href="#systems-thinking-questions" title="Link to this heading">#</a></h2>
|
||
<section id="real-world-applications">
|
||
<h3>Real-World Applications<a class="headerlink" href="#real-world-applications" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Deep Learning Training</strong>: All neural network layers operate on tensors - Linear layers perform matrix multiplication, Conv2d applies tensor convolutions, Attention mechanisms compute tensor dot products. How would doubling model size affect memory and compute requirements?</p></li>
|
||
<li><p><strong>Computer Vision</strong>: Images are 3D tensors (height × width × channels), and every transformation (resize, crop, normalize) is a tensor operation. What’s the memory footprint of a batch of 32 images at 224×224 resolution with 3 color channels in float32?</p></li>
|
||
<li><p><strong>Natural Language Processing</strong>: Text embeddings are 2D tensors (sequence_length × embedding_dim), and Transformer models manipulate these through attention. For BERT with 512 sequence length and 768 hidden dimension, how many elements per sample?</p></li>
|
||
<li><p><strong>Scientific Computing</strong>: Tensors represent multidimensional data in climate models, molecular simulations, physics engines. What makes tensors more efficient than nested Python lists for these applications?</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="mathematical-foundations">
|
||
<h3>Mathematical Foundations<a class="headerlink" href="#mathematical-foundations" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Linear Algebra</strong>: Tensors generalize matrices to arbitrary dimensions. How does broadcasting relate to outer products? When is <code class="docutils literal notranslate"><span class="pre">(M,K)</span> <span class="pre">@</span> <span class="pre">(K,N)</span></code> more efficient than <code class="docutils literal notranslate"><span class="pre">(K,M).T</span> <span class="pre">@</span> <span class="pre">(K,N)</span></code>?</p></li>
|
||
<li><p><strong>Numerical Stability</strong>: Operations like softmax require careful implementation to avoid overflow/underflow. Why does <code class="docutils literal notranslate"><span class="pre">exp(x</span> <span class="pre">-</span> <span class="pre">max(x))</span></code> prevent overflow in softmax computation?</p></li>
|
||
<li><p><strong>Broadcasting Semantics</strong>: NumPy’s broadcasting rules enable elegant code but require understanding shape compatibility. Can you predict the output shape of <code class="docutils literal notranslate"><span class="pre">(32,</span> <span class="pre">1,</span> <span class="pre">10)</span> <span class="pre">+</span> <span class="pre">(1,</span> <span class="pre">5,</span> <span class="pre">10)</span></code>?</p></li>
|
||
<li><p><strong>Computational Complexity</strong>: Matrix multiplication is O(n³) while element-wise operations are O(n). For large models, which dominates training time and why?</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="performance-characteristics">
|
||
<h3>Performance Characteristics<a class="headerlink" href="#performance-characteristics" title="Link to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Memory Contiguity</strong>: Contiguous memory enables SIMD vectorization and cache efficiency. How much can non-contiguous tensors slow down operations (10x? 100x?)?</p></li>
|
||
<li><p><strong>View vs Copy</strong>: Views are O(1) with shared memory, copies are O(n) with duplicated storage. When might a view cause unexpected behavior (e.g., in-place operations)?</p></li>
|
||
<li><p><strong>Operation Fusion</strong>: Frameworks optimize <code class="docutils literal notranslate"><span class="pre">(a</span> <span class="pre">+</span> <span class="pre">b)</span> <span class="pre">*</span> <span class="pre">c</span></code> by fusing operations to reduce memory reads. How many memory passes does unfused require vs. fused?</p></li>
|
||
<li><p><strong>Batch Processing</strong>: Processing 32 images at once is much faster than 32 sequential passes. Why? (Hint: GPU parallelism, cache reuse, reduced Python overhead)</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="whats-next">
|
||
<h2>What’s Next<a class="headerlink" href="#whats-next" title="Link to this heading">#</a></h2>
|
||
<p>After mastering tensors, you’re ready to build the computational layers of neural networks:</p>
|
||
<p><strong>Module 02: Activations</strong> - Implement ReLU, Sigmoid, Tanh, and Softmax activation functions that introduce non-linearity. You’ll operate on your Tensor class and understand why activation functions are essential for learning complex patterns.</p>
|
||
<p><strong>Module 03: Layers</strong> - Build Linear (fully-connected) and convolutional layers using tensor operations. See how weight matrices and bias vectors (stored as Tensors) transform inputs through matrix multiplication and broadcasting.</p>
|
||
<p><strong>Module 05: Autograd</strong> - Add automatic differentiation to your Tensor class, enabling gradient computation for training. Your tensors will track operations and compute gradients automatically - the magic behind <code class="docutils literal notranslate"><span class="pre">loss.backward()</span></code>.</p>
|
||
<p><strong>Preview of tensor usage ahead:</strong></p>
|
||
<ul class="simple">
|
||
<li><p>Activations: <code class="docutils literal notranslate"><span class="pre">output</span> <span class="pre">=</span> <span class="pre">ReLU()(input_tensor)</span></code> - element-wise operations on tensors</p></li>
|
||
<li><p>Layers: <code class="docutils literal notranslate"><span class="pre">output</span> <span class="pre">=</span> <span class="pre">Linear(in_features=128,</span> <span class="pre">out_features=64)(input_tensor)</span></code> - matmul with weight tensors</p></li>
|
||
<li><p>Loss: <code class="docutils literal notranslate"><span class="pre">loss</span> <span class="pre">=</span> <span class="pre">MSELoss()(predictions,</span> <span class="pre">targets)</span></code> - tensor reductions for error measurement</p></li>
|
||
<li><p>Training: <code class="docutils literal notranslate"><span class="pre">optimizer.step()</span></code> updates parameter tensors using gradients</p></li>
|
||
</ul>
|
||
<p>Every module builds on your Tensor foundation - understanding tensors deeply means understanding how neural networks actually compute.</p>
|
||
</section>
|
||
<section id="ready-to-build">
|
||
<h2>Ready to Build?<a class="headerlink" href="#ready-to-build" title="Link to this heading">#</a></h2>
|
||
<p>You’re about to implement the foundation of all machine learning systems! The Tensor class you’ll build is the universal data structure that powers everything from simple neural networks to GPT, Stable Diffusion, and AlphaFold.</p>
|
||
<p>This is where mathematical abstraction meets practical implementation. You’ll see how N-dimensional arrays enable elegant representations of complex data, how operator overloading makes tensor math feel natural like <code class="docutils literal notranslate"><span class="pre">z</span> <span class="pre">=</span> <span class="pre">x</span> <span class="pre">+</span> <span class="pre">y</span></code>, and how careful memory management (views vs. copies) enables working with massive models. Every decision you make - from how to handle broadcasting to when to validate shapes - reflects trade-offs that production ML engineers face daily.</p>
|
||
<p>Take your time with this module. Understand each operation deeply. Test your implementations thoroughly. The Tensor foundation you build here will support every subsequent module - if you understand tensors from first principles, you’ll understand how neural networks actually work, not just how to use them.</p>
|
||
<p>Every neural network you’ve ever used - ResNet, BERT, GPT, Stable Diffusion - is fundamentally built on tensor operations. Understanding tensors means understanding the computational substrate of modern AI.</p>
|
||
<p>Choose your preferred way to engage with this module:</p>
|
||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||
<div class="sd-row sd-row-cols-1 sd-row-cols-xs-1 sd-row-cols-sm-2 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
🚀 Launch Binder</div>
|
||
<p class="sd-card-text">Run this module interactively in your browser. No installation required!</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://mybinder.org/v2/gh/mlsysbook/TinyTorch/main?filepath=modules/01_tensor/tensor_dev.ipynb"><span>https://mybinder.org/v2/gh/mlsysbook/TinyTorch/main?filepath=modules/01_tensor/tensor_dev.ipynb</span></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
⚡ Open in Colab</div>
|
||
<p class="sd-card-text">Use Google Colab for GPU access and cloud compute power.</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://colab.research.google.com/github/mlsysbook/TinyTorch/blob/main/modules/01_tensor/tensor_dev.ipynb"><span>https://colab.research.google.com/github/mlsysbook/TinyTorch/blob/main/modules/01_tensor/tensor_dev.ipynb</span></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
📖 View Source</div>
|
||
<p class="sd-card-text">Browse the Jupyter notebook and understand the implementation.</p>
|
||
</div>
|
||
<a class="sd-stretched-link sd-hide-link-text reference external" href="https://github.com/mlsysbook/TinyTorch/blob/main/modules/01_tensor/tensor_dev.ipynb"><span>https://github.com/mlsysbook/TinyTorch/blob/main/modules/01_tensor/tensor_dev.ipynb</span></a></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">💾 Save Your Progress</p>
|
||
<p><strong>Binder sessions are temporary!</strong> Download your completed notebook when done, or switch to local development for persistent work.</p>
|
||
</div>
|
||
<hr class="docutils" />
|
||
<div class="prev-next-area">
|
||
<a class="right-next" href="../02_activations/ABOUT.html" title="next page">Next Module →</a>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./modules"
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="../tiers/foundation.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">🏗 Foundation Tier (Modules 01-07)</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="02_activations_ABOUT.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">02. Activations</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-youll-build">What You’ll Build</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#core-concepts">Core Concepts</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensors-as-multidimensional-arrays">Tensors as Multidimensional Arrays</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#broadcasting-efficient-shape-alignment">Broadcasting: Efficient Shape Alignment</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#views-vs-copies-memory-efficiency">Views vs. Copies: Memory Efficiency</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#computational-complexity">Computational Complexity</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#architecture-overview">Architecture Overview</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensor-class-design">Tensor Class Design</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#data-flow-architecture">Data Flow Architecture</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#module-integration">Module Integration</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tensor-class-foundation">Tensor Class Foundation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#arithmetic-operations">Arithmetic Operations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#matrix-multiplication">Matrix Multiplication</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#shape-manipulation">Shape Manipulation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reduction-operations">Reduction Operations</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#production-context">Production Context</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#your-implementation-vs-production-frameworks">Your Implementation vs. Production Frameworks</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#side-by-side-code-comparison">Side-by-Side Code Comparison</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-production-usage">Real-World Production Usage</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics-at-scale">Performance Characteristics at Scale</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#package-integration">Package Integration</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-your-implementation-maps-to-pytorch">How Your Implementation Maps to PyTorch</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#common-pitfalls">Common Pitfalls</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#shape-mismatch-errors">Shape Mismatch Errors</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#broadcasting-confusion">Broadcasting Confusion</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#view-vs-copy-confusion">View vs Copy Confusion</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#axis-parameter-mistakes">Axis Parameter Mistakes</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dtype-issues">Dtype Issues</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-leaks-with-large-tensors">Memory Leaks with Large Tensors</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-applications">Real-World Applications</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-foundations">Mathematical Foundations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics">Performance Characteristics</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#whats-next">What’s Next</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-build">Ready to Build?</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Prof. Vijay Janapa Reddi (Harvard University)
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2025.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |