mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-09 10:39:41 -06:00
117 lines
5.2 KiB
Python
117 lines
5.2 KiB
Python
"""Regenerate the executable calibration report, or check it without writing.
|
||
|
||
Run from mlsysim/: python scripts/calibration_report.py [--check]
|
||
Uses the installed package; install this checkout in editable mode first.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
from pathlib import Path
|
||
|
||
from mlsysim.engine.empirical import EMPIRICAL_ANCHORS
|
||
from mlsysim.physics import calc_transformer_training_flops
|
||
from mlsysim.solvers import ServingModel, SingleNodeModel
|
||
|
||
DESTINATION = Path(__file__).resolve().parents[1] / "docs" / "empirical-calibration.md"
|
||
|
||
|
||
def render() -> str:
|
||
rows = []
|
||
for anchor in EMPIRICAL_ANCHORS:
|
||
if anchor.solver == "SingleNodeModel":
|
||
value = SingleNodeModel().solve(anchor.workload, anchor.hardware, **anchor.solver_kwargs).throughput
|
||
category = "Calibrated regression"
|
||
elif anchor.solver == "ServingModel":
|
||
value = ServingModel().solve(anchor.workload, anchor.hardware, **anchor.solver_kwargs).itl
|
||
category = "Family sanity range"
|
||
elif anchor.solver == "calc_transformer_training_flops":
|
||
value = calc_transformer_training_flops(anchor.workload.parameters, anchor.workload.training_tokens)
|
||
category = "Formula accounting"
|
||
else:
|
||
raise ValueError(f"Unsupported calibration solver: {anchor.solver}")
|
||
predicted = value.m_as(anchor.units)
|
||
target = (f"{float(anchor.target):.6g}" if anchor.target is not None
|
||
else f"{float(anchor.lower):.6g}–{float(anchor.upper):.6g}")
|
||
agreement = (f"{(predicted / float(anchor.target) - 1) * 100:+.2f}%"
|
||
if anchor.target is not None else "within range" if anchor.accepts(predicted) else "outside range")
|
||
rows.append(f"| `{anchor.id}` | {category} | {predicted:.6g} {anchor.units} | {target} {anchor.units} | {agreement} |")
|
||
configs = "\n".join(f"- **{a.id}:** {a.configuration}. {a.review_notes}" for a in EMPIRICAL_ANCHORS)
|
||
return '''# Calibration and validation scope
|
||
|
||
<!-- Generated by scripts/calibration_report.py; edit the script or anchor definitions. -->
|
||
|
||
MLSysIM estimates are analytical outputs, not hardware measurements. The table
|
||
below is generated from `engine.empirical`; it distinguishes calibrated checks,
|
||
broad sanity ranges, and formula accounting. Passing these checks does not
|
||
establish an accuracy percentage for unseen workloads.
|
||
|
||
| Anchor | Evidence category | Analytical output | Reference | Difference |
|
||
|---|---|---|---|---|
|
||
''' + "\n".join(rows) + '''
|
||
|
||
## Executed configurations
|
||
|
||
''' + configs + '''
|
||
|
||
## What the comparisons establish
|
||
|
||
The ResNet efficiencies are calibration inputs, not independently predicted
|
||
utilization. The H100 reference is a per-accelerator result from a distributed
|
||
training run, not a matched single-device experiment. The Llama interval is a
|
||
family sanity range; it does not identify one engine, precision, and request
|
||
shape. The GPT-3 comparison checks FLOP accounting, not wall-clock performance.
|
||
Source identities and limitations are recorded in the provenance catalog.
|
||
|
||
For an external accuracy claim, pin the source revision, hardware topology,
|
||
model, engine, precision, batch/concurrency, input/output lengths, and metric.
|
||
Retain the raw timing artifact and declare efficiency assumptions before the
|
||
comparison. Evaluate held-out configurations separately from calibration.
|
||
At batch one, decode ITL is `(end_to_end_latency - TTFT) / (output_tokens - 1)`;
|
||
end-to-end latency divided by output tokens includes prefill and is a different
|
||
metric. A TTFT prediction must not be checked against an ITL reference.
|
||
|
||
## Modeling conventions
|
||
|
||
- `efficiency` derates compute FLOP/s. It does not derate memory bandwidth.
|
||
- Serving decode uses the larger of compute and weight/KV memory time, plus
|
||
the configured layer tax. Continuous batching uses the same step accounting.
|
||
- Prefill is a compute-based estimate, with attention work and dispatch taxes;
|
||
very short prompts can be memory- or overhead-limited in practice.
|
||
- A weight-only bandwidth floor remains useful, but is not a complete latency
|
||
prediction or a serving capacity guarantee.
|
||
- Runtime workspaces, scheduling interference, and inter-device communication
|
||
require separate modeling or measurement. Feasible means the modeled state
|
||
fits; it does not reserve all memory used by a production engine.
|
||
|
||
## Reproduce
|
||
|
||
From an editable checkout's `mlsysim/` directory:
|
||
|
||
```bash
|
||
python scripts/calibration_report.py --check
|
||
python -m pytest tests/test_empirical.py
|
||
```
|
||
|
||
Run the script without `--check` to regenerate this file. Numerical outputs and
|
||
reference differences are computed by the script, not entered by hand.
|
||
'''
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--check", action="store_true")
|
||
args = parser.parse_args()
|
||
content = render()
|
||
if args.check:
|
||
if not DESTINATION.exists() or DESTINATION.read_text() != content:
|
||
print("Calibration report is stale; run python scripts/calibration_report.py")
|
||
return 1
|
||
print("Calibration report matches executable anchors.")
|
||
else:
|
||
DESTINATION.write_text(content)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|