mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-05-29 16:18:16 -05:00
1105 lines
74 KiB
HTML
1105 lines
74 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
|
||
|
||
<title>11. Embeddings - Token to Vector Representations — Tiny🔥Torch</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="../_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="../_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="../_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||
<link rel="stylesheet" href="../_static/styles/sphinx-book-theme.css?digest=14f4ca6b54d191a8c7657f6c759bf11a5fb86285" type="text/css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/mystnb.4510f1fc1dee50b3e5859aac5469c37c29e427902b24a333a5f9fcb2f0b3ac41.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/custom.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/design-style.4045f2051d55cab465a707391d5b2007.min.css" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="../_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||
<script src="../_static/jquery.js"></script>
|
||
<script src="../_static/underscore.js"></script>
|
||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||
<script src="../_static/doctools.js"></script>
|
||
<script src="../_static/clipboard.min.js"></script>
|
||
<script src="../_static/copybutton.js"></script>
|
||
<script src="../_static/scripts/sphinx-book-theme.js?digest=5a5c038af52cf7bc1a1ec88eea08e6366ee68824"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../_static/togglebutton.js"></script>
|
||
<script src="../_static/wip-banner.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="../_static/design-tabs.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="../_static/sphinx-thebe.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'modules/11_embeddings_ABOUT';</script>
|
||
<link rel="shortcut icon" href="../_static/favicon.svg"/>
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
<link rel="next" title="12. Attention - The Mechanism That Powers Modern AI" href="12_attention_ABOUT.html" />
|
||
<link rel="prev" title="10. Tokenization - Text to Numerical Sequences" href="10_tokenization_ABOUT.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="../search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../_static/logo-tinytorch.png" class="logo__image only-light" alt="Tiny🔥Torch - Home"/>
|
||
<script>document.write(`<img src="../_static/logo-tinytorch.png" class="logo__image only-dark" alt="Tiny🔥Torch - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="../intro.html">
|
||
Welcome
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../quickstart-guide.html">Quick Start Guide</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../student-workflow.html">Student Workflow</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../usage-paths/classroom-use.html">For Instructors</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏗 Foundation Tier (01-07)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="01_tensor_ABOUT.html">01. Tensor</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="02_activations_ABOUT.html">02. Activations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="03_layers_ABOUT.html">03. Layers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="04_losses_ABOUT.html">04. Losses</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="05_autograd_ABOUT.html">05. Autograd</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="06_optimizers_ABOUT.html">06. Optimizers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="07_training_ABOUT.html">07. Training</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏛️ Architecture Tier (08-13)</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="08_dataloader_ABOUT.html">08. DataLoader</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="09_spatial_ABOUT.html">09. Convolutions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="10_tokenization_ABOUT.html">10. Tokenization</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">11. Embeddings</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="12_attention_ABOUT.html">12. Attention</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="13_transformers_ABOUT.html">13. Transformers</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">⏱️ Optimization Tier (14-19)</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="14_profiling_ABOUT.html">14. Profiling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="15_quantization_ABOUT.html">15. Quantization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="16_compression_ABOUT.html">16. Compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="17_memoization_ABOUT.html">17. Memoization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="18_acceleration_ABOUT.html">18. Acceleration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="19_benchmarking_ABOUT.html">19. Benchmarking</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">🏅 Capstone Competition</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="20_capstone_ABOUT.html">20. Torch Olympics</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Course Orientation</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/00-introduction.html">Course Structure</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../prerequisites.html">Prerequisites & Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/learning-journey.html">Learning Journey</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../chapters/milestones.html">Historical Milestones</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Using TinyTorch</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../tito-essentials.html">Essential Commands</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../datasets.html">Datasets Guide</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../testing-framework.html">Testing Guide</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Community</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../community.html">Ecosystem</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../credits.html">Credits & Acknowledgments</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources.html">Additional Resources</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><label class="sidebar-toggle primary-toggle btn btn-sm" for="__primary" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</label></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-source-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Source repositories">
|
||
<i class="fab fa-github"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch" target="_blank"
|
||
class="btn btn-sm btn-source-repository-button dropdown-item"
|
||
title="Source repository"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fab fa-github"></i>
|
||
</span>
|
||
<span class="btn__text-container">Repository</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch/edit/main/book/modules/11_embeddings_ABOUT.md" target="_blank"
|
||
class="btn btn-sm btn-source-edit-button dropdown-item"
|
||
title="Suggest edit"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-pencil-alt"></i>
|
||
</span>
|
||
<span class="btn__text-container">Suggest edit</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li><a href="https://github.com/mlsysbook/TinyTorch/issues/new?title=Issue%20on%20page%20%2Fmodules/11_embeddings_ABOUT.html&body=Your%20issue%20content%20here." target="_blank"
|
||
class="btn btn-sm btn-source-issues-button dropdown-item"
|
||
title="Open an issue"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-lightbulb"></i>
|
||
</span>
|
||
<span class="btn__text-container">Open issue</span>
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="../_sources/modules/11_embeddings_ABOUT.md" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.md</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<label class="sidebar-toggle secondary-toggle btn btn-sm" for="__secondary"title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</label>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>11. Embeddings - Token to Vector Representations</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#embedding-layer-the-token-lookup-table">Embedding Layer - The Token Lookup Table</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#sinusoidal-positional-encoding-transformer-style">Sinusoidal Positional Encoding (Transformer-Style)</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#learned-positional-encoding-gpt-style">Learned Positional Encoding (GPT-Style)</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-embedding-system">Complete Embedding System</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-applications">Real-World Applications</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-foundations">Mathematical Foundations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics">Performance Characteristics</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-build">Ready to Build?</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section id="embeddings-token-to-vector-representations">
|
||
<h1>11. Embeddings - Token to Vector Representations<a class="headerlink" href="#embeddings-token-to-vector-representations" title="Permalink to this heading">#</a></h1>
|
||
<p><strong>ARCHITECTURE TIER</strong> | Difficulty: ⭐⭐ (2/4) | Time: 4-5 hours</p>
|
||
<section id="overview">
|
||
<h2>Overview<a class="headerlink" href="#overview" title="Permalink to this heading">#</a></h2>
|
||
<p>Build the embedding systems that transform discrete token IDs into dense, learnable vector representations - the bridge between symbolic text and neural computation. This module implements lookup tables, positional encodings, and the optimization techniques that power every modern language model from word2vec to GPT-4’s input layers.</p>
|
||
<p>You’ll discover why embeddings aren’t just “lookup tables” but sophisticated parameter spaces where semantic meaning emerges through training. By implementing both token embeddings and positional encodings from scratch, you’ll understand the architectural choices that shape how transformers process language and why certain design decisions (sinusoidal vs learned positions, embedding dimensions, initialization strategies) have profound implications for model capacity, memory usage, and inference performance.</p>
|
||
</section>
|
||
<section id="learning-objectives">
|
||
<h2>Learning Objectives<a class="headerlink" href="#learning-objectives" title="Permalink to this heading">#</a></h2>
|
||
<p>By the end of this module, you will be able to:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Implement embedding layers</strong>: Build efficient lookup tables for token-to-vector conversion with proper Xavier initialization and gradient flow</p></li>
|
||
<li><p><strong>Design positional encodings</strong>: Create both sinusoidal (Transformer-style) and learned (GPT-style) position representations with different extrapolation capabilities</p></li>
|
||
<li><p><strong>Understand memory scaling</strong>: Analyze how vocabulary size and embedding dimensions impact parameter count, memory bandwidth, and serving costs</p></li>
|
||
<li><p><strong>Optimize embedding lookups</strong>: Implement sparse gradient updates that avoid computing gradients for 99% of vocabulary during training</p></li>
|
||
<li><p><strong>Apply dimensionality principles</strong>: Balance semantic expressiveness with computational efficiency in vector space design and initialization</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="build-use-reflect">
|
||
<h2>Build → Use → Reflect<a class="headerlink" href="#build-use-reflect" title="Permalink to this heading">#</a></h2>
|
||
<p>This module follows TinyTorch’s <strong>Build → Use → Reflect</strong> framework:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Build</strong>: Implement embedding lookup tables with trainable parameters, sinusoidal positional encodings using mathematical patterns, learned position embeddings, and complete token+position combination systems</p></li>
|
||
<li><p><strong>Use</strong>: Convert tokenized text sequences to dense vectors, add positional information for sequence order awareness, and prepare embeddings for attention mechanisms</p></li>
|
||
<li><p><strong>Reflect</strong>: Analyze memory scaling with vocabulary size (why GPT-3’s embeddings use 2.4GB), understand sparse gradient efficiency for large vocabularies, and explore semantic geometry in learned embedding spaces</p></li>
|
||
</ol>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">Systems Reality Check</p>
|
||
<p><strong>Production Context</strong>: GPT-3’s embedding table contains 50,257 vocabulary × 12,288 dimensions = 617M parameters (about 20% of the model’s 175B total). Every token lookup requires reading 48KB of memory - making embedding access a major bandwidth bottleneck during inference, especially for long sequences.</p>
|
||
<p><strong>Performance Note</strong>: During training, only ~1% of vocabulary appears in each batch. Sparse gradient updates avoid computing gradients for the other 99% of embedding parameters, saving massive computation and memory bandwidth. This is why frameworks like PyTorch implement specialized sparse gradient operations for embeddings.</p>
|
||
</div>
|
||
</section>
|
||
<section id="implementation-guide">
|
||
<h2>Implementation Guide<a class="headerlink" href="#implementation-guide" title="Permalink to this heading">#</a></h2>
|
||
<section id="embedding-layer-the-token-lookup-table">
|
||
<h3>Embedding Layer - The Token Lookup Table<a class="headerlink" href="#embedding-layer-the-token-lookup-table" title="Permalink to this heading">#</a></h3>
|
||
<p>The fundamental building block that maps discrete token IDs to continuous dense vectors. This is where semantic meaning will eventually be learned through training.</p>
|
||
<p><strong>Core Implementation Pattern:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">Embedding</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""Learnable embedding layer for token-to-vector conversion.</span>
|
||
|
||
<span class="sd"> Implements efficient lookup table that maps token IDs to dense vectors.</span>
|
||
<span class="sd"> The foundation of all language models and sequence processing.</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> vocab_size: Size of vocabulary (e.g., 50,000 for GPT-2)</span>
|
||
<span class="sd"> embedding_dim: Dimension of dense vectors (e.g., 768 for BERT-base)</span>
|
||
|
||
<span class="sd"> Memory Cost: vocab_size × embedding_dim parameters</span>
|
||
<span class="sd"> Example: 50K vocab × 768 dim = 38.4M parameters (153MB at FP32)</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vocab_size</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">embedding_dim</span> <span class="o">=</span> <span class="n">embedding_dim</span>
|
||
|
||
<span class="c1"># Xavier/Glorot initialization for stable gradients</span>
|
||
<span class="n">limit</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="mf">6.0</span> <span class="o">/</span> <span class="p">(</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="n">embedding_dim</span><span class="p">))</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="n">limit</span><span class="p">,</span> <span class="n">limit</span><span class="p">,</span> <span class="p">(</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">)),</span>
|
||
<span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Look up embeddings for token IDs.</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> indices: (batch_size, seq_len) tensor of token IDs</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> embeddings: (batch_size, seq_len, embedding_dim) dense vectors</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Advanced indexing: O(1) per token lookup</span>
|
||
<span class="n">embedded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">indices</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)]</span>
|
||
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">embedded</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Attach gradient computation (sparse updates during backward)</span>
|
||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">:</span>
|
||
<span class="n">result</span><span class="o">.</span><span class="n">_grad_fn</span> <span class="o">=</span> <span class="n">EmbeddingBackward</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="p">,</span> <span class="n">indices</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">result</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why This Design Works:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Xavier initialization</strong> ensures gradients don’t explode or vanish during early training</p></li>
|
||
<li><p><strong>Advanced indexing</strong> provides O(1) lookup complexity regardless of vocabulary size</p></li>
|
||
<li><p><strong>Sparse gradients</strong> mean only embeddings for tokens in the current batch receive updates</p></li>
|
||
<li><p><strong>Trainable weights</strong> allow the model to learn semantic relationships through backpropagation</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="sinusoidal-positional-encoding-transformer-style">
|
||
<h3>Sinusoidal Positional Encoding (Transformer-Style)<a class="headerlink" href="#sinusoidal-positional-encoding-transformer-style" title="Permalink to this heading">#</a></h3>
|
||
<p>Fixed mathematical encodings that capture position without learned parameters. The original “Attention is All You Need” approach that enables extrapolation to longer sequences.</p>
|
||
<p><strong>Mathematical Foundation:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">create_sinusoidal_embeddings</span><span class="p">(</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Create sinusoidal positional encodings from Vaswani et al. (2017).</span>
|
||
|
||
<span class="sd"> Uses sine/cosine functions of different frequencies to encode position.</span>
|
||
|
||
<span class="sd"> Formula:</span>
|
||
<span class="sd"> PE(pos, 2i) = sin(pos / 10000^(2i/embed_dim)) # Even indices</span>
|
||
<span class="sd"> PE(pos, 2i+1) = cos(pos / 10000^(2i/embed_dim)) # Odd indices</span>
|
||
|
||
<span class="sd"> Where:</span>
|
||
<span class="sd"> pos = position in sequence (0, 1, 2, ...)</span>
|
||
<span class="sd"> i = dimension pair index</span>
|
||
<span class="sd"> 10000 = base frequency (creates wavelengths from 2π to 10000·2π)</span>
|
||
|
||
<span class="sd"> Advantages:</span>
|
||
<span class="sd"> - Zero parameters (no memory overhead)</span>
|
||
<span class="sd"> - Generalizes to sequences longer than training</span>
|
||
<span class="sd"> - Smooth transitions (nearby positions similar)</span>
|
||
<span class="sd"> - Rich frequency spectrum across dimensions</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Position indices: [0, 1, 2, ..., max_seq_len-1]</span>
|
||
<span class="n">position</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
|
||
|
||
<span class="c1"># Frequency term for each dimension pair</span>
|
||
<span class="n">div_term</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span> <span class="o">*</span>
|
||
<span class="o">-</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">10000.0</span><span class="p">)</span> <span class="o">/</span> <span class="n">embedding_dim</span><span class="p">)</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="c1"># Initialize positional encoding matrix</span>
|
||
<span class="n">pe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Apply sine to even indices (0, 2, 4, ...)</span>
|
||
<span class="n">pe</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">::</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="n">position</span> <span class="o">*</span> <span class="n">div_term</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Apply cosine to odd indices (1, 3, 5, ...)</span>
|
||
<span class="n">pe</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">::</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">position</span> <span class="o">*</span> <span class="n">div_term</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">pe</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Why Sinusoidal Patterns Work:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Different frequencies</strong> per dimension: high frequencies change rapidly between positions, low frequencies change slowly</p></li>
|
||
<li><p><strong>Unique signatures</strong> for each position through combination of frequencies</p></li>
|
||
<li><p><strong>Linear combinations</strong> allow the model to learn relative position offsets through attention</p></li>
|
||
<li><p><strong>No length limit</strong> - can compute encodings for any sequence length at inference time</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="learned-positional-encoding-gpt-style">
|
||
<h3>Learned Positional Encoding (GPT-Style)<a class="headerlink" href="#learned-positional-encoding-gpt-style" title="Permalink to this heading">#</a></h3>
|
||
<p>Trainable position embeddings that can adapt to task-specific patterns. Used in GPT models and other architectures where positional patterns may be learnable.</p>
|
||
<p><strong>Implementation Pattern:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">PositionalEncoding</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""Learnable positional encoding layer.</span>
|
||
|
||
<span class="sd"> Trainable position-specific vectors added to token embeddings.</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> max_seq_len: Maximum sequence length to support</span>
|
||
<span class="sd"> embedding_dim: Dimension matching token embeddings</span>
|
||
|
||
<span class="sd"> Advantages:</span>
|
||
<span class="sd"> - Can learn task-specific position patterns</span>
|
||
<span class="sd"> - May capture regularities like sentence structure</span>
|
||
<span class="sd"> - Often performs slightly better than sinusoidal</span>
|
||
|
||
<span class="sd"> Disadvantages:</span>
|
||
<span class="sd"> - Requires additional parameters (max_seq_len × embedding_dim)</span>
|
||
<span class="sd"> - Cannot extrapolate beyond training sequence length</span>
|
||
<span class="sd"> - Needs sufficient training data to learn position patterns</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">max_seq_len</span> <span class="o">=</span> <span class="n">max_seq_len</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">embedding_dim</span> <span class="o">=</span> <span class="n">embedding_dim</span>
|
||
|
||
<span class="c1"># Smaller initialization than token embeddings (additive combination)</span>
|
||
<span class="n">limit</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="mf">2.0</span> <span class="o">/</span> <span class="n">embedding_dim</span><span class="p">)</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">position_embeddings</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="n">limit</span><span class="p">,</span> <span class="n">limit</span><span class="p">,</span> <span class="p">(</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">)),</span>
|
||
<span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Add positional encodings to input embeddings.</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> x: (batch_size, seq_len, embedding_dim) input embeddings</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> Position-aware embeddings of same shape</span>
|
||
<span class="sd"> """</span>
|
||
<span class="n">batch_size</span><span class="p">,</span> <span class="n">seq_len</span><span class="p">,</span> <span class="n">embedding_dim</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span>
|
||
|
||
<span class="c1"># Get position embeddings for this sequence length</span>
|
||
<span class="n">pos_embeddings</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">position_embeddings</span><span class="o">.</span><span class="n">data</span><span class="p">[:</span><span class="n">seq_len</span><span class="p">]</span>
|
||
|
||
<span class="c1"># Broadcast to batch dimension: (1, seq_len, embedding_dim)</span>
|
||
<span class="n">pos_embeddings</span> <span class="o">=</span> <span class="n">pos_embeddings</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="p">:,</span> <span class="p">:]</span>
|
||
|
||
<span class="c1"># Element-wise addition combines token and position information</span>
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">pos_embeddings</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">result</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Design Rationale:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Learned parameters</strong> can capture task-specific patterns (e.g., sentence beginnings, clause boundaries)</p></li>
|
||
<li><p><strong>Smaller initialization</strong> because positions add to token embeddings (not replace them)</p></li>
|
||
<li><p><strong>Fixed max length</strong> is a limitation but acceptable for many production use cases</p></li>
|
||
<li><p><strong>Element-wise addition</strong> preserves both token semantics and position information</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="complete-embedding-system">
|
||
<h3>Complete Embedding System<a class="headerlink" href="#complete-embedding-system" title="Permalink to this heading">#</a></h3>
|
||
<p>Production-ready integration of token and positional embeddings used in real transformer implementations.</p>
|
||
<p><strong>Full Pipeline:</strong></p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">EmbeddingLayer</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""Complete embedding system combining token and positional embeddings.</span>
|
||
|
||
<span class="sd"> Production component matching PyTorch/HuggingFace transformer patterns.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">max_seq_len</span><span class="o">=</span><span class="mi">512</span><span class="p">,</span>
|
||
<span class="n">pos_encoding</span><span class="o">=</span><span class="s1">'learned'</span><span class="p">,</span> <span class="n">scale_embeddings</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">scale_embeddings</span> <span class="o">=</span> <span class="n">scale_embeddings</span>
|
||
|
||
<span class="c1"># Token embedding table</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">token_embedding</span> <span class="o">=</span> <span class="n">Embedding</span><span class="p">(</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Positional encoding strategy</span>
|
||
<span class="k">if</span> <span class="n">pos_encoding</span> <span class="o">==</span> <span class="s1">'learned'</span><span class="p">:</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">pos_encoding</span> <span class="o">=</span> <span class="n">PositionalEncoding</span><span class="p">(</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
|
||
<span class="k">elif</span> <span class="n">pos_encoding</span> <span class="o">==</span> <span class="s1">'sinusoidal'</span><span class="p">:</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">pos_encoding</span> <span class="o">=</span> <span class="n">create_sinusoidal_embeddings</span><span class="p">(</span><span class="n">max_seq_len</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
|
||
<span class="k">elif</span> <span class="n">pos_encoding</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">pos_encoding</span> <span class="o">=</span> <span class="kc">None</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tokens</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Convert tokens to position-aware embeddings.</span>
|
||
|
||
<span class="sd"> Args:</span>
|
||
<span class="sd"> tokens: (batch_size, seq_len) token indices</span>
|
||
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> (batch_size, seq_len, embed_dim) position-aware vectors</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Token lookup</span>
|
||
<span class="n">token_embeds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">token_embedding</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Optional scaling (Transformer convention: √embed_dim)</span>
|
||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_embeddings</span><span class="p">:</span>
|
||
<span class="n">token_embeds</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">token_embeds</span><span class="o">.</span><span class="n">data</span> <span class="o">*</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">))</span>
|
||
|
||
<span class="c1"># Add positional information</span>
|
||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos_encoding</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos_encoding</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">token_embeds</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">output</span> <span class="o">=</span> <span class="n">token_embeds</span>
|
||
|
||
<span class="k">return</span> <span class="n">output</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Integration Benefits:</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Flexible positional encoding</strong> supports learned, sinusoidal, or none</p></li>
|
||
<li><p><strong>Embedding scaling</strong> (multiply by √d) is Transformer convention for gradient stability</p></li>
|
||
<li><p><strong>Batch processing</strong> handles variable sequence lengths efficiently</p></li>
|
||
<li><p><strong>Parameter management</strong> tracks all trainable components for optimization</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="getting-started">
|
||
<h2>Getting Started<a class="headerlink" href="#getting-started" title="Permalink to this heading">#</a></h2>
|
||
<section id="prerequisites">
|
||
<h3>Prerequisites<a class="headerlink" href="#prerequisites" title="Permalink to this heading">#</a></h3>
|
||
<p>Before starting this module, ensure you have completed:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Module 01 (Tensor)</strong>: Provides the foundational Tensor class with gradient tracking and operations</p></li>
|
||
<li><p><strong>Module 10 (Tokenization)</strong>: Required for converting text to token IDs that embeddings consume</p></li>
|
||
</ul>
|
||
<p>Verify your prerequisites:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Activate TinyTorch environment</span>
|
||
<span class="nb">source</span><span class="w"> </span>bin/activate-tinytorch.sh
|
||
|
||
<span class="c1"># Verify prerequisite modules</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>--module<span class="w"> </span>tensor
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>--module<span class="w"> </span>tokenization
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="development-workflow">
|
||
<h3>Development Workflow<a class="headerlink" href="#development-workflow" title="Permalink to this heading">#</a></h3>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Open the development notebook</strong>: <code class="docutils literal notranslate"><span class="pre">modules/11_embeddings/embeddings_dev.ipynb</span></code></p></li>
|
||
<li><p><strong>Implement Embedding class</strong>: Create lookup table with Xavier initialization and efficient indexing</p></li>
|
||
<li><p><strong>Build sinusoidal encodings</strong>: Compute sine/cosine position representations using mathematical formula</p></li>
|
||
<li><p><strong>Create learned positions</strong>: Add trainable position embedding table with proper initialization</p></li>
|
||
<li><p><strong>Integrate complete system</strong>: Combine token and position embeddings with flexible encoding strategies</p></li>
|
||
<li><p><strong>Export and verify</strong>: <code class="docutils literal notranslate"><span class="pre">tito</span> <span class="pre">module</span> <span class="pre">complete</span> <span class="pre">11</span> <span class="pre">&&</span> <span class="pre">tito</span> <span class="pre">test</span> <span class="pre">--module</span> <span class="pre">embeddings</span></code></p></li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="testing">
|
||
<h2>Testing<a class="headerlink" href="#testing" title="Permalink to this heading">#</a></h2>
|
||
<section id="comprehensive-test-suite">
|
||
<h3>Comprehensive Test Suite<a class="headerlink" href="#comprehensive-test-suite" title="Permalink to this heading">#</a></h3>
|
||
<p>Run the full test suite to verify embedding functionality:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># TinyTorch CLI (recommended)</span>
|
||
tito<span class="w"> </span><span class="nb">test</span><span class="w"> </span>--module<span class="w"> </span>embeddings
|
||
|
||
<span class="c1"># Direct pytest execution</span>
|
||
python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>tests/<span class="w"> </span>-k<span class="w"> </span>embeddings<span class="w"> </span>-v
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="test-coverage-areas">
|
||
<h3>Test Coverage Areas<a class="headerlink" href="#test-coverage-areas" title="Permalink to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p>✅ <strong>Embedding lookup correctness</strong>: Verify token IDs map to correct vector rows in weight table</p></li>
|
||
<li><p>✅ <strong>Gradient flow verification</strong>: Ensure sparse gradient updates accumulate properly during backpropagation</p></li>
|
||
<li><p>✅ <strong>Positional encoding math</strong>: Validate sinusoidal formula implementation with correct frequencies</p></li>
|
||
<li><p>✅ <strong>Shape broadcasting</strong>: Test token + position combination across batch dimensions</p></li>
|
||
<li><p>✅ <strong>Memory efficiency profiling</strong>: Verify parameter count and lookup performance characteristics</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="inline-testing-validation">
|
||
<h3>Inline Testing & Validation<a class="headerlink" href="#inline-testing-validation" title="Permalink to this heading">#</a></h3>
|
||
<p>The module includes comprehensive unit tests during development:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span># Example inline test output
|
||
🔬 Unit Test: Embedding layer...
|
||
✅ Lookup table created: 10K vocab × 256 dims = 2.56M parameters
|
||
✅ Forward pass shape correct: (32, 20, 256) for batch of 32 sequences
|
||
✅ Backward pass sparse gradients accumulate correctly
|
||
✅ Xavier initialization keeps variance stable
|
||
📈 Progress: Embedding Layer ✓
|
||
|
||
🔬 Unit Test: Sinusoidal positional encoding...
|
||
✅ Encodings computed for 512 positions × 256 dimensions
|
||
✅ Sine/cosine patterns verified (pos 0: [0, 1, 0, 1, ...])
|
||
✅ Different positions have unique signatures
|
||
✅ Frequency spectrum correct (high to low across dimensions)
|
||
📈 Progress: Sinusoidal Positions ✓
|
||
|
||
🔬 Unit Test: Learned positional encoding...
|
||
✅ Trainable position embeddings initialized
|
||
✅ Addition with token embeddings preserves gradients
|
||
✅ Batch broadcasting handled correctly
|
||
📈 Progress: Learned Positions ✓
|
||
|
||
🔬 Unit Test: Complete embedding system...
|
||
✅ Token + position combination works for all strategies
|
||
✅ Embedding scaling (√d) applied correctly
|
||
✅ Variable sequence lengths handled gracefully
|
||
✅ Parameter counting correct for each configuration
|
||
📈 Progress: Complete System ✓
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="manual-testing-examples">
|
||
<h3>Manual Testing Examples<a class="headerlink" href="#manual-testing-examples" title="Permalink to this heading">#</a></h3>
|
||
<p>Test your embedding implementation interactively:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">tinytorch.text.embeddings</span><span class="w"> </span><span class="kn">import</span> <span class="n">Embedding</span><span class="p">,</span> <span class="n">PositionalEncoding</span><span class="p">,</span> <span class="n">create_sinusoidal_embeddings</span>
|
||
|
||
<span class="c1"># Create embedding layer</span>
|
||
<span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span> <span class="o">=</span> <span class="mi">10000</span><span class="p">,</span> <span class="mi">256</span>
|
||
<span class="n">token_emb</span> <span class="o">=</span> <span class="n">Embedding</span><span class="p">(</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test token lookup</span>
|
||
<span class="n">token_ids</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">23</span><span class="p">],</span> <span class="p">[</span><span class="mi">42</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">19</span><span class="p">]])</span> <span class="c1"># (2, 3) - batch of 2 sequences</span>
|
||
<span class="n">embeddings</span> <span class="o">=</span> <span class="n">token_emb</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">token_ids</span><span class="p">)</span> <span class="c1"># (2, 3, 256)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Token embeddings shape: </span><span class="si">{</span><span class="n">embeddings</span><span class="o">.</span><span class="n">shape</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Add learned positional encodings</span>
|
||
<span class="n">pos_emb</span> <span class="o">=</span> <span class="n">PositionalEncoding</span><span class="p">(</span><span class="n">max_seq_len</span><span class="o">=</span><span class="mi">512</span><span class="p">,</span> <span class="n">embed_dim</span><span class="o">=</span><span class="mi">256</span><span class="p">)</span>
|
||
<span class="n">token_embeddings_3d</span> <span class="o">=</span> <span class="n">embeddings</span> <span class="c1"># Already (batch, seq, embed)</span>
|
||
<span class="n">pos_aware</span> <span class="o">=</span> <span class="n">pos_emb</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">token_embeddings_3d</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Position-aware shape: </span><span class="si">{</span><span class="n">pos_aware</span><span class="o">.</span><span class="n">shape</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="c1"># (2, 3, 256)</span>
|
||
|
||
<span class="c1"># Try sinusoidal encodings</span>
|
||
<span class="n">sin_pe</span> <span class="o">=</span> <span class="n">create_sinusoidal_embeddings</span><span class="p">(</span><span class="n">max_seq_len</span><span class="o">=</span><span class="mi">512</span><span class="p">,</span> <span class="n">embed_dim</span><span class="o">=</span><span class="mi">256</span><span class="p">)</span>
|
||
<span class="n">sin_positions</span> <span class="o">=</span> <span class="n">sin_pe</span><span class="o">.</span><span class="n">data</span><span class="p">[:</span><span class="mi">3</span><span class="p">][</span><span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="p">:,</span> <span class="p">:]</span> <span class="c1"># (1, 3, 256)</span>
|
||
<span class="n">combined</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">embeddings</span><span class="o">.</span><span class="n">data</span> <span class="o">+</span> <span class="n">sin_positions</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sinusoidal combined: </span><span class="si">{</span><span class="n">combined</span><span class="o">.</span><span class="n">shape</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="c1"># (2, 3, 256)</span>
|
||
|
||
<span class="c1"># Verify position 0 pattern (should be [0, 1, 0, 1, ...])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Position 0 pattern: </span><span class="si">{</span><span class="n">sin_pe</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="w"> </span><span class="p">:</span><span class="mi">8</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="c1"># Expected: [~0.0, ~1.0, ~0.0, ~1.0, ~0.0, ~1.0, ~0.0, ~1.0]</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="systems-thinking-questions">
|
||
<h2>Systems Thinking Questions<a class="headerlink" href="#systems-thinking-questions" title="Permalink to this heading">#</a></h2>
|
||
<section id="real-world-applications">
|
||
<h3>Real-World Applications<a class="headerlink" href="#real-world-applications" title="Permalink to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Large Language Models (GPT-4, Claude, Llama)</strong>: Embedding tables often contain 20-40% of total model parameters. GPT-3’s 50K vocab × 12K dims = 617M embedding parameters alone (2.4GB at FP32). This makes embeddings a major memory consumer in serving infrastructure.</p></li>
|
||
<li><p><strong>Recommendation Systems (YouTube, Netflix, Spotify)</strong>: Billion-scale item embeddings for personalized content retrieval. YouTube’s embedding space contains hundreds of millions of video embeddings, enabling fast nearest-neighbor search for recommendations in milliseconds.</p></li>
|
||
<li><p><strong>Multilingual Models (Google Translate, mBERT)</strong>: Shared embedding spaces across 100+ languages enable zero-shot cross-lingual transfer. Words with similar meanings across languages cluster together in the learned vector space, allowing translation without parallel data.</p></li>
|
||
<li><p><strong>Search Engines (Google, Bing)</strong>: Query and document embeddings power semantic search beyond keyword matching. BERT-style embeddings capture meaning, letting “how to fix a leaky faucet” match “plumbing repair for dripping tap” even with no shared words.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="mathematical-foundations">
|
||
<h3>Mathematical Foundations<a class="headerlink" href="#mathematical-foundations" title="Permalink to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Embedding Geometry</strong>: Why do word embeddings exhibit linear relationships like “king - man + woman ≈ queen”? The training objective (predicting context words in word2vec, or next tokens in language models) creates geometric structure where semantic relationships become linear vector operations. This emerges without explicit supervision.</p></li>
|
||
<li><p><strong>Dimensionality Trade-offs</strong>: Higher dimensions increase expressiveness (more capacity to separate distinct concepts) but require more memory and computation. BERT-base uses 768 dimensions, BERT-large uses 1024 - carefully chosen based on performance-cost Pareto analysis. Doubling dimensions doubles memory but may only improve accuracy by a few percentage points.</p></li>
|
||
<li><p><strong>Positional Encoding Mathematics</strong>: Sinusoidal encodings use different frequencies (wavelengths from 2π to 10,000·2π) so each position gets a unique pattern. The model can learn relative positions through attention: the dot product of position encodings at offsets k captures periodic patterns the attention mechanism learns to use.</p></li>
|
||
<li><p><strong>Sparse Gradient Efficiency</strong>: During training with vocabulary size V and batch containing b unique tokens, dense gradients would update all V embeddings. Sparse gradients only update b embeddings - when b << V (typical: 1000 tokens vs 50K vocab), this saves ~98% of gradient computation and memory bandwidth.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="performance-characteristics">
|
||
<h3>Performance Characteristics<a class="headerlink" href="#performance-characteristics" title="Permalink to this heading">#</a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Memory Scaling</strong>: Embedding tables grow as O(vocab_size × embedding_dim). At FP32 (4 bytes per parameter): 50K vocab × 768 dims = 153MB, 100K vocab × 1024 dims = 410MB. Mixed precision (FP16) cuts this in half, but vocabulary size dominates scaling for large models.</p></li>
|
||
<li><p><strong>Bandwidth Bottleneck</strong>: Every token lookup reads embedding_dim × sizeof(dtype) bytes from memory. With 768 dims at FP32, that’s 3KB per token. Processing a 2048-token context requires reading 6MB from the embedding table - memory bandwidth becomes the bottleneck, not compute.</p></li>
|
||
<li><p><strong>Cache Efficiency</strong>: Sequential token access has poor cache locality because tokens are typically non-sequential in the embedding table (token IDs [1, 42, 7, 99] means random jumps through the weight matrix). Batching improves throughput by amortizing cache misses, but embedding access remains memory-bound, not compute-bound.</p></li>
|
||
<li><p><strong>Inference Optimization</strong>: Embedding quantization (INT8 or even INT4) reduces memory footprint and bandwidth by 2-4×, critical for deployment. KV-caching in transformers makes embedding lookup happen only once per token (not per layer), so optimizing this cold start is important for latency-sensitive applications.</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="ready-to-build">
|
||
<h2>Ready to Build?<a class="headerlink" href="#ready-to-build" title="Permalink to this heading">#</a></h2>
|
||
<p>You’re about to implement the embedding systems that power modern AI language understanding. These lookup tables and positional encodings are the bridge between discrete tokens (words, subwords, characters) and the continuous vector spaces where neural networks operate. What seems like a simple “array lookup” is actually the foundation of how language models represent meaning.</p>
|
||
<p>What makes this module special is understanding not just <em>how</em> embeddings work, but <em>why</em> certain design choices matter. Why do we need positional encodings when embeddings already contain token information? Why sparse gradients instead of dense updates? How does embedding dimension affect model capacity versus memory footprint? These aren’t just implementation details - they’re fundamental design principles that shape every production language model’s architecture.</p>
|
||
<p>By building embeddings from scratch, you’ll gain intuition for memory-computation trade-offs in deep learning systems. You’ll understand why GPT-3’s embedding table consumes 2.4GB of memory, and why that matters for serving costs at scale (more memory = more expensive GPUs = higher operational costs). You’ll see how sinusoidal encodings allow transformers to process sequences longer than training data, while learned positions might perform better on specific tasks with known maximum lengths.</p>
|
||
<p>This is where theory meets the economic realities of deploying AI at scale. Every architectural choice - vocabulary size, embedding dimension, positional encoding strategy - has both technical implications (accuracy, generalization) and business implications (memory costs, inference latency, serving throughput). Understanding these trade-offs is what separates machine learning researchers from machine learning systems engineers.</p>
|
||
<p>Choose your preferred way to engage with this module:</p>
|
||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||
<div class="sd-row sd-row-cols-1 sd-row-cols-xs-1 sd-row-cols-sm-2 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
🚀 Launch Binder</div>
|
||
<p class="sd-card-text">Run this module interactively in your browser. No installation required!</p>
|
||
</div>
|
||
<a class="sd-stretched-link reference external" href="https://mybinder.org/v2/gh/mlsysbook/TinyTorch/main?filepath=modules/11_embeddings/embeddings_dev.ipynb"></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
⚡ Open in Colab</div>
|
||
<p class="sd-card-text">Use Google Colab for GPU access and cloud compute power.</p>
|
||
</div>
|
||
<a class="sd-stretched-link reference external" href="https://colab.research.google.com/github/mlsysbook/TinyTorch/blob/main/modules/11_embeddings/embeddings_dev.ipynb"></a></div>
|
||
</div>
|
||
<div class="sd-col sd-d-flex-row docutils">
|
||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||
<div class="sd-card-body docutils">
|
||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||
📖 View Source</div>
|
||
<p class="sd-card-text">Browse the Jupyter notebook source and understand the implementation.</p>
|
||
</div>
|
||
<a class="sd-stretched-link reference external" href="https://github.com/mlsysbook/TinyTorch/blob/main/modules/11_embeddings/embeddings_dev.ipynb"></a></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tip admonition">
|
||
<p class="admonition-title">💾 Save Your Progress</p>
|
||
<p><strong>Binder sessions are temporary!</strong> Download your completed notebook when done, or switch to local development for persistent work.</p>
|
||
</div>
|
||
<hr class="docutils" />
|
||
<div class="prev-next-area">
|
||
<a class="left-prev" href="../modules/10_tokenization_ABOUT.html" title="previous page">← Previous Module</a>
|
||
<a class="right-next" href="../modules/12_attention_ABOUT.html" title="next page">Next Module →</a>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./modules"
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="10_tokenization_ABOUT.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">10. Tokenization - Text to Numerical Sequences</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="12_attention_ABOUT.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">12. Attention - The Mechanism That Powers Modern AI</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview">Overview</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-objectives">Learning Objectives</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build-use-reflect">Build → Use → Reflect</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guide">Implementation Guide</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#embedding-layer-the-token-lookup-table">Embedding Layer - The Token Lookup Table</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#sinusoidal-positional-encoding-transformer-style">Sinusoidal Positional Encoding (Transformer-Style)</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#learned-positional-encoding-gpt-style">Learned Positional Encoding (GPT-Style)</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#complete-embedding-system">Complete Embedding System</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#getting-started">Getting Started</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites">Prerequisites</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#development-workflow">Development Workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing">Testing</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comprehensive-test-suite">Comprehensive Test Suite</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-coverage-areas">Test Coverage Areas</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#inline-testing-validation">Inline Testing & Validation</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#manual-testing-examples">Manual Testing Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#systems-thinking-questions">Systems Thinking Questions</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#real-world-applications">Real-World Applications</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-foundations">Mathematical Foundations</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-characteristics">Performance Characteristics</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ready-to-build">Ready to Build?</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Prof. Vijay Janapa Reddi (Harvard University)
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2025.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="../_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |