mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-16 12:05:12 -06:00
151 lines
4.9 KiB
Python
151 lines
4.9 KiB
Python
"""Vol I–IV calculations must stay wired to MLSysIM registries.
|
||
|
||
The book is allowed to have illustrative scenario constants in a chapter, but
|
||
model, hardware, dataset, platform, system, infrastructure, literature, and ops
|
||
objects used by rendered calculations should resolve through the package API.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from pathlib import Path
|
||
|
||
import mlsysim
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||
BOOK_ROOTS = tuple(
|
||
REPO_ROOT / "books" / f"vol{i}" for i in range(1, 5)
|
||
)
|
||
|
||
REGISTRY_ROOTS = (
|
||
"Models",
|
||
"Hardware",
|
||
"Systems",
|
||
"Infrastructure",
|
||
"Platforms",
|
||
"Datasets",
|
||
"Literature",
|
||
"Scenarios",
|
||
"ReferenceStats",
|
||
"Ops",
|
||
"Sensors",
|
||
"Actuators",
|
||
"Embodied",
|
||
"Agents",
|
||
)
|
||
REGISTRY_PATH_RE = re.compile(
|
||
r"\b(" + "|".join(REGISTRY_ROOTS) + r")(?:\.[A-Za-z_]\w*)+"
|
||
)
|
||
|
||
# Quarto examples often call methods after a registry value, e.g.
|
||
# ``Hardware.Cloud.H100.memory.capacity.m_as(GB)``. Resolve the registry object
|
||
# and stop before the Pint/Python method suffix.
|
||
METHOD_TAILS = frozenset({
|
||
"format_benchmark_table",
|
||
"items",
|
||
"keys",
|
||
"m",
|
||
"m_as",
|
||
"magnitude",
|
||
"to",
|
||
"values",
|
||
})
|
||
|
||
SHORTHAND_SUFFIXES = frozenset({"{", "*"})
|
||
|
||
|
||
def _qmd_files() -> list[Path]:
|
||
files: list[Path] = []
|
||
for root in BOOK_ROOTS:
|
||
files.extend(sorted(root.rglob("*.qmd")))
|
||
return files
|
||
|
||
|
||
def _resolve_registry_ref(ref: str) -> object:
|
||
obj: object = mlsysim
|
||
for part in ref.split("."):
|
||
if part in METHOD_TAILS:
|
||
break
|
||
obj = getattr(obj, part)
|
||
return obj
|
||
|
||
|
||
def test_all_volume_registry_paths_resolve() -> None:
|
||
missing: list[str] = []
|
||
|
||
for path in _qmd_files():
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
rel = path.relative_to(REPO_ROOT)
|
||
|
||
for match in REGISTRY_PATH_RE.finditer(text):
|
||
# Ignore comment shorthand such as ``Hardware.Foo.{A,B}``; concrete
|
||
# paths beside it are still scanned and resolved.
|
||
if text[match.end():match.end() + 1] in SHORTHAND_SUFFIXES:
|
||
continue
|
||
|
||
ref = match.group(0)
|
||
try:
|
||
_resolve_registry_ref(ref)
|
||
except AttributeError as exc:
|
||
missing.append(f"{rel}: {ref} ({exc})")
|
||
|
||
assert not missing, (
|
||
"Vol I–IV QMD registry references must resolve through mlsysim:\n"
|
||
+ "\n".join(missing[:80])
|
||
+ (f"\n... and {len(missing) - 80} more" if len(missing) > 80 else "")
|
||
)
|
||
|
||
|
||
def test_book_case_study_model_specs_are_registry_backed() -> None:
|
||
"""High-signal textbook model anchors should live under Models.*."""
|
||
|
||
lenet = mlsysim.Models.Vision.LeNet1
|
||
mixtral = mlsysim.Models.Language.Mixtral_8x7B
|
||
|
||
assert lenet.parameters.m_as("param") == 10_000
|
||
assert mixtral.parameters.m_as("Bparam") == 46.7
|
||
assert mixtral.active_parameters.m_as("Bparam") == 12.9
|
||
assert mixtral.experts == 8
|
||
assert mixtral.active_experts_per_token == 2
|
||
|
||
|
||
def test_scenarios_are_executable_model_system_constraint_bundles() -> None:
|
||
"""Concrete scenarios compose existing registries rather than redefining facts."""
|
||
|
||
assert not hasattr(mlsysim.Scenarios, "Archetypes")
|
||
|
||
assert mlsysim.Scenarios.FrontierTraining.workload is mlsysim.Models.Language.GPT4
|
||
assert mlsysim.Scenarios.FrontierTraining.system is mlsysim.Systems.Clusters.Frontier_8K
|
||
|
||
assert mlsysim.Scenarios.AutonomousVehicle.workload is mlsysim.Models.Vision.YOLOv8_Nano
|
||
assert mlsysim.Scenarios.AutonomousVehicle.system is mlsysim.Hardware.Edge.JetsonOrinNX
|
||
|
||
assert mlsysim.Scenarios.MobileAssistant.workload is mlsysim.Models.Language.Llama3_8B
|
||
assert mlsysim.Scenarios.MobileAssistant.system is mlsysim.Hardware.Mobile.iPhone15Pro
|
||
|
||
assert mlsysim.Scenarios.SmartDoorbell.workload is mlsysim.Models.Tiny.WakeVision
|
||
assert mlsysim.Scenarios.SmartDoorbell.system is mlsysim.Hardware.Tiny.ESP32_S3
|
||
|
||
|
||
def test_reference_stats_are_separate_from_executable_scenarios() -> None:
|
||
"""Sourced anchors stay under ReferenceStats, not executable Scenarios."""
|
||
|
||
assert hasattr(mlsysim.ReferenceStats, "Workloads")
|
||
assert hasattr(mlsysim.ReferenceStats, "MobilePower")
|
||
assert hasattr(mlsysim.ReferenceStats, "PhoneBattery")
|
||
assert hasattr(mlsysim.ReferenceStats, "OuraSleepStudy")
|
||
assert not hasattr(mlsysim.Scenarios, "ReferenceStats")
|
||
assert not hasattr(mlsysim.Scenarios, "Workloads")
|
||
assert not hasattr(mlsysim.Scenarios, "MobilePower")
|
||
|
||
|
||
def test_no_legacy_scenario_aliases_remain() -> None:
|
||
"""Scenarios and reference statistics should have one public path each."""
|
||
|
||
assert not hasattr(mlsysim, "Applications")
|
||
assert not hasattr(mlsysim, "ScenarioBundles")
|
||
assert not hasattr(mlsysim.Scenarios, "Doorbell")
|
||
assert not hasattr(mlsysim.Scenarios, "AutoDrive")
|
||
assert not hasattr(mlsysim.Scenarios, "Frontier")
|
||
assert not hasattr(mlsysim.reference_stats.registry, "Scenarios")
|