github-starred/cs249r_book
2
0
Fork 0
mirror of https://github.com/harvard-edge/cs249r_book.git synced 2026-07-25 05:50:17 -05:00
Code Issues 276 Packages Projects Releases 26 Wiki Activity

Migrate Vol I remaining chapters from .m_as() to .to().magnitude.

Browse Source 
Covers model_compression through conclusion plus Vol I appendices; fixes
byte-vs-Bparam and missing USD/kg/param imports surfaced by exec tests.
This commit is contained in:
Vijay Janapa Reddi
2026-05-31 18:58:47 -04:00
parent 9548af10e4
commit 0fa88d8715
12 changed files with 471 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files
book/quarto/contents/vol1/backmatter/appendix_algorithm.qmd
+35 -35
View File
@@ -78,11 +78,11 @@ class GemmIntensityExample:
n_small = 64
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
bytes_fp16 = int(BYTES_FP16.m_as(ureg.byte))
bytes_fp16 = int(BYTES_FP16.to(byte).magnitude)
small_intensity = n_small / 3
a100_ridge = int(
Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
/ Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
/ Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
)
small_efficiency_pct = small_intensity / a100_ridge * 100
@@ -140,14 +140,14 @@ class SparseEmbeddingExample:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
total_elements = vocab_size * embed_dim
dense_bytes_q = total_elements * BYTES_FP32
dense_bytes = dense_bytes_q.m_as(byte)
dense_gb = dense_bytes_q.m_as(GB)
dense_bytes = dense_bytes_q.to(byte).magnitude
dense_gb = dense_bytes_q.to(GB).magnitude
nonzeros = int(total_elements * sparsity_pct / 100)
csr_bytes_per_nonzero_q = BYTES_FP32 + BYTES_INT32
csr_bytes_per_nonzero = csr_bytes_per_nonzero_q.m_as(byte)
csr_bytes_per_nonzero = csr_bytes_per_nonzero_q.to(byte).magnitude
sparse_bytes_q = nonzeros * csr_bytes_per_nonzero_q
sparse_bytes = sparse_bytes_q.m_as(byte)
sparse_mb = sparse_bytes_q.m_as(MB)
sparse_bytes = sparse_bytes_q.to(byte).magnitude
sparse_mb = sparse_bytes_q.to(MB).magnitude
reduction_factor = int(dense_bytes / sparse_bytes)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -157,8 +157,8 @@ class SparseEmbeddingExample:
dense_gb_str = fmt_qty(dense_bytes_q, GB, precision=0, commas=False)
nonzeros_str = fmt(nonzeros / MILLION, precision=0, commas=False)
sparse_mb_str = fmt_qty(sparse_bytes_q, MB, precision=0, commas=False)
bytes_fp32_str = fmt_int(BYTES_FP32.m_as(ureg.byte), commas=False)
bytes_int32_str = fmt_int(BYTES_INT32.m_as(ureg.byte), commas=False)
bytes_fp32_str = fmt_int(BYTES_FP32.to(byte).magnitude, commas=False)
bytes_int32_str = fmt_int(BYTES_INT32.to(byte).magnitude, commas=False)
sparsity_pct_str = fmt_percent(sparsity_pct/100, precision=0, commas=False, style='prose')
reduction_factor_str = fmt(reduction_factor, precision=0, commas=False)
```
@@ -314,10 +314,10 @@ from mlsysim.fmt import fmt_int, fmt, MarkdownStr
class TrainingMemoryBytes:
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
bytes_fp16_str = fmt_int(BYTES_FP16.m_as(ureg.byte), commas=False)
bytes_fp32_str = fmt_int(BYTES_FP32.m_as(ureg.byte), commas=False)
optimizer_overhead_min = int(BYTES_ADAM_STATE.m_as(ureg.byte))
optimizer_overhead_max = optimizer_overhead_min + int(BYTES_FP32.m_as(ureg.byte))
bytes_fp16_str = fmt_int(BYTES_FP16.to(byte).magnitude, commas=False)
bytes_fp32_str = fmt_int(BYTES_FP32.to(byte).magnitude, commas=False)
optimizer_overhead_min = int(BYTES_ADAM_STATE.to(byte).magnitude)
optimizer_overhead_max = optimizer_overhead_min + int(BYTES_FP32.to(byte).magnitude)
optimizer_overhead_str = MarkdownStr(f"{optimizer_overhead_min}–{optimizer_overhead_max}")
```
@@ -351,48 +351,48 @@ To see how these components interact in practice, consider a concrete model.
# │ gpt2_act_small_gb_str, gpt2_total_small_gb_str,
# │ gpt2_batch_large_str, gpt2_act_large_gb_str
# └─────────────────────────────────────────────────────────────────────────────
from mlsysim.fmt import fmt_int, fmt, check, sci_latex, fmt_math, MarkdownStr, fmt_percent, fmt_qty
from mlsysim.fmt import fmt_int, fmt, check, sci_latex, fmt_math, fmt_percent, fmt_qty
class GPT2TrainingMem:
# ┌── 1. LOAD ──────────────────────────────────────────
P = Models.Language.GPT2.parameters.m_as(param)
P = Models.Language.GPT2.parameters.to(param).magnitude
layers = Models.Language.GPT2.layers
d = Models.Language.GPT2.hidden_dim
bf16 = int(BYTES_FP16.m_as(ureg.byte))
fp32 = int(BYTES_FP32.m_as(ureg.byte))
bf16 = int(BYTES_FP16.to(byte).magnitude)
fp32 = int(BYTES_FP32.to(byte).magnitude)
accel_q = Hardware.Cloud.A100.memory.capacity
accel_gb = accel_q.m_as(GB)
accel_gb = accel_q.to(GB).magnitude
batch_small = 8
batch_large = 64
seq_len = 1024
optimizer_bytes_per_param = int(
(BYTES_ADAM_STATE + BYTES_FP32).m_as(ureg.byte)
(BYTES_ADAM_STATE + BYTES_FP32).to(byte).magnitude
) # FP32 master + momentum + variance
# ┌── 2. EXECUTE ───────────────────────────────────────
weights_q = P * bf16 * byte
grads_q = P * bf16 * byte
optim_q = P * optimizer_bytes_per_param * byte
weights_gb = weights_q.m_as(GB)
grads_gb = grads_q.m_as(GB)
optim_gb = optim_q.m_as(GB)
weights_gb = weights_q.to(GB).magnitude
grads_gb = grads_q.to(GB).magnitude
optim_gb = optim_q.to(GB).magnitude
model_state_q = weights_q + grads_q + optim_q
model_state_gb = model_state_q.m_as(GB)
model_state_gb = model_state_q.to(GB).magnitude
remaining_q = accel_q - model_state_q
remaining_gb = remaining_q.m_as(GB)
remaining_gb = remaining_q.to(GB).magnitude
# Per-layer activation ≈ 12 * B * S * d * bytes_bf16:
# input(1d) + QKV(3d) + attn_out(1d) + FFN(4d) + output(1d) + norms/masks(2d)
act_factor = 12
act_small_q = layers * act_factor * batch_small * seq_len * d * bf16 * byte
act_small_bytes = act_small_q.m_as(byte)
act_small_gb = act_small_q.m_as(GB)
act_small_bytes = act_small_q.to(byte).magnitude
act_small_gb = act_small_q.to(GB).magnitude
total_small_q = model_state_q + act_small_q
total_small_gb = total_small_q.m_as(GB)
total_small_gb = total_small_q.to(GB).magnitude
act_large_q = layers * act_factor * batch_large * seq_len * d * bf16 * byte
act_large_bytes = act_large_q.m_as(byte)
act_large_gb = act_large_q.m_as(GB)
act_large_bytes = act_large_q.to(byte).magnitude
act_large_gb = act_large_q.to(GB).magnitude
# ┌── 3. GUARD ─────────────────────────────────────────
check(model_state_gb < accel_gb, "Model state must fit on accelerator")
@@ -400,7 +400,7 @@ class GPT2TrainingMem:
check(act_large_gb > remaining_gb, "Large batch should exceed remaining capacity")
# ┌── 4. OUTPUT ────────────────────────────────────────
P_str = MarkdownStr(f"${sci_latex(P, precision=1)}$") # LaTeX-safe: 1.5 \times 10^{9} (avoids 1.5e+09)
P_str = fmt_math(sci_latex(P, precision=1)) # LaTeX-safe: 1.5 \times 10^{9} (avoids 1.5e+09)
layers_str = fmt(layers, precision=0, commas=False)
d_str = fmt(d, precision=0, commas=False)
bf16_str = fmt(bf16, precision=0, commas=False)
@@ -490,10 +490,10 @@ from mlsysim.fmt import fmt
class SparseBreakEvenRecap:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
csr_bytes_per_nonzero = BYTES_FP32.m_as(ureg.byte) + BYTES_INT32.m_as(ureg.byte)
coo_bytes_per_nonzero = BYTES_FP32.m_as(ureg.byte) + 2 * BYTES_INT32.m_as(ureg.byte)
csr_break_even_density_pct = BYTES_FP32.m_as(ureg.byte) / csr_bytes_per_nonzero * 100
coo_break_even_density_pct = BYTES_FP32.m_as(ureg.byte) / coo_bytes_per_nonzero * 100
csr_bytes_per_nonzero = BYTES_FP32.to(byte).magnitude + BYTES_INT32.to(byte).magnitude
coo_bytes_per_nonzero = BYTES_FP32.to(byte).magnitude + 2 * BYTES_INT32.to(byte).magnitude
csr_break_even_density_pct = BYTES_FP32.to(byte).magnitude / csr_bytes_per_nonzero * 100
coo_break_even_density_pct = BYTES_FP32.to(byte).magnitude / coo_bytes_per_nonzero * 100
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
csr_break_even_density_pct_str = fmt_percent(csr_break_even_density_pct/100, precision=0, commas=False, style='prose')
book/quarto/contents/vol1/backmatter/appendix_assumptions.qmd
+8 -8
View File
@@ -51,13 +51,13 @@ class NapkinMath:
h100_bw_q = Hardware.Cloud.H100.memory.bandwidth
h100_mem_q = Hardware.Cloud.H100.memory.capacity
a100_tdp_q = Hardware.Cloud.A100.tdp
h100_tflops = h100_flops_q.m_as(TFLOPs/second)
h100_bw_tbs = h100_bw_q.m_as(TB/second)
h100_mem_gb = h100_mem_q.m_as(GB)
a100_tdp_w = a100_tdp_q.m_as(watt)
h100_tflops = h100_flops_q.to(TFLOPs/second).magnitude
h100_bw_tbs = h100_bw_q.to(TB/second).magnitude
h100_mem_gb = h100_mem_q.to(GB).magnitude
a100_tdp_w = a100_tdp_q.to(watt).magnitude
gpt3_ops = Models.Language.GPT3.training_ops.magnitude
gpt3_days = Models.Language.GPT3.training_days_ref.magnitude
gpt3_accelerators = Models.Language.GPT3.training_accelerators_ref.m_as(count)
gpt3_accelerators = Models.Language.GPT3.training_accelerators_ref.to(count).magnitude
elec_per_kwh = Infrastructure.Pricing.Cloud.ElectricityPerKwh.rate.magnitude
gemm_n = 4096
@@ -67,15 +67,15 @@ class NapkinMath:
+ BYTES_FP16 # BF16 gradients
+ 3 * BYTES_FP32 # FP32 master weights + momentum + variance
)
bytes_per_param = bytes_per_param_q.m_as(byte)
bytes_per_param = bytes_per_param_q.to(byte).magnitude
hours_per_day = HOURS_PER_DAY
# ┌── 2. EXECUTE ───────────────────────────────────────
ridge = h100_tflops/h100_bw_tbs
gemm_intensity = gemm_n / 3
train_mem_q = model_params_b * BILLION * bytes_per_param_q
train_mem_gb = train_mem_q.m_as(GB)
a100_tdp_kw = a100_tdp_q.m_as(kilowatt)
train_mem_gb = train_mem_q.to(GB).magnitude
a100_tdp_kw = a100_tdp_q.to(kW).magnitude
elec_cost = gpt3_days * gpt3_accelerators * hours_per_day * a100_tdp_kw * elec_per_kwh
# ┌── 3. GUARD ─────────────────────────────────────────
book/quarto/contents/vol1/backmatter/appendix_data.qmd
+13 -13
View File
@@ -53,11 +53,11 @@ from mlsysim import Systems # Ethernet fabric bandwidths
class DataGravity:
# ┌── 1. LOAD (Constants) ──────────────────────────────
tb_bytes = (1 * TB).m_as(byte)
pb_bytes = (1 * PB).m_as(byte)
bw_1g = (1 * Gbps).m_as('bit/second')
bw_10g = Systems.Fabrics.Ethernet_10G.bandwidth.m_as('bit/second')
bw_100g = Systems.Fabrics.Ethernet_100G.bandwidth.m_as('bit/second')
tb_bytes = (1 * TB).to(byte).magnitude
pb_bytes = (1 * PB).to(byte).magnitude
bw_1g = (1 * Gbps).to(bit / second).magnitude
bw_10g = Systems.Fabrics.Ethernet_10G.bandwidth.to(bit / second).magnitude
bw_100g = Systems.Fabrics.Ethernet_100G.bandwidth.to(bit / second).magnitude
truck_load_hours = 8
truck_transit_hours = 32
truck_unload_hours = 8
@@ -159,12 +159,12 @@ class SerializationCost:
# ┌── 1. LOAD (Constants) ──────────────────────────────
csv_speed = 100 * MB/second
parquet_speed = 1000 * MB/second
csv_speed_mb = csv_speed.m_as(MB/second)
parquet_speed_mb = parquet_speed.m_as(MB/second)
csv_speed_mb = csv_speed.to(MB/second).magnitude
parquet_speed_mb = parquet_speed.to(MB/second).magnitude
csv_cycles = 100
parquet_cycles = 10
proto_speed = 300 * MB/second
proto_speed_mb = proto_speed.m_as(MB/second)
proto_speed_mb = proto_speed.to(MB/second).magnitude
proto_cycles = 200
# ┌── 2. EXECUTE (The Compute) ─────────────────────────
@@ -332,16 +332,16 @@ from mlsysim.fmt import fmt, check, fmt_percent, fmt_qty, MarkdownStr
class DataAlgebra:
# ┌── 1. LOAD (Constants) ──────────────────────────────
row_size_q = 1 * KB
row_size_kb = row_size_q.m_as(KB)
int_size_bytes = BYTES_INT32.m_as(byte)
row_size_kb = row_size_q.to(KB).magnitude
int_size_bytes = BYTES_INT32.to(byte).magnitude
join_table_q = 1 * TB
join_table_tb = join_table_q.m_as(TB)
join_table_tb = join_table_q.to(TB).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────
row_size_bytes = row_size_q.m_as(byte)
row_size_bytes = row_size_q.to(byte).magnitude
waste_pct = (1 - (int_size_bytes / row_size_bytes)) * 100
join_network_q = 2 * join_table_q
join_network_tb = join_network_q.m_as(TB)
join_network_tb = join_network_q.to(TB).magnitude
# ┌── 3. GUARD (Invariants) ────────────────────────────
check(waste_pct > 99, "Row-format waste should be >99%")
book/quarto/contents/vol1/backmatter/appendix_machine.qmd
+62 -62
View File
@@ -89,45 +89,45 @@ class NumbersToKnow:
LATENCY_HBM3 = Hardware.Tech.Memory.DRAM.latency
LATENCY_PCIE_GEN5 = Hardware.Tech.Interconnect.PCIeGen5.latency
LATENCY_NVME_SSD = Hardware.Tech.Storage.NvmeGen4.latency
LATENCY_INFINIBAND = round(Systems.Fabrics.InfiniBand_NDR.latency.m_as(NS)) * NS # µs→ns, round to exact 5000
LATENCY_INFINIBAND = round(Systems.Fabrics.InfiniBand_NDR.latency.to(NS).magnitude) * NS # µs→ns, round to exact 5000
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
# Step 1: Physics Constants (Eternal)
speed_of_light_km_ms = int(SPEED_OF_LIGHT_FIBER_KM_S.m_as(ureg.kilometer / ureg.millisecond))
speed_of_light_km_ms = int(SPEED_OF_LIGHT_FIBER_KM_S.to(km / ms).magnitude)
# Step 2: Energy Ratios (Stable across process nodes)
dram_vs_compute = int(ENERGY_DRAM_ACCESS_PJ.m_as(ureg.picojoule) / ENERGY_FLOP_FP16_PJ.m_as(ureg.picojoule))
fp32_vs_int8 = int(ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule) / ENERGY_OP_INT8_PJ.m_as(ureg.picojoule))
fp32_vs_fp16 = round(ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule) / ENERGY_FLOP_FP16_PJ.m_as(ureg.picojoule), 1)
l1_vs_reg = int(ENERGY_SRAM_L1_PJ.m_as(ureg.picojoule) / ENERGY_REG_PJ.m_as(ureg.picojoule))
dram_vs_compute = int(ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude / ENERGY_FLOP_FP16_PJ.to(pJ).magnitude)
fp32_vs_int8 = int(ENERGY_FLOP_FP32_PJ.to(pJ).magnitude / ENERGY_OP_INT8_PJ.to(pJ).magnitude)
fp32_vs_fp16 = round(ENERGY_FLOP_FP32_PJ.to(pJ).magnitude / ENERGY_FLOP_FP16_PJ.to(pJ).magnitude, 1)
l1_vs_reg = int(ENERGY_SRAM_L1_PJ.to(pJ).magnitude / ENERGY_REG_PJ.to(pJ).magnitude)
# Step 3: Memory Hierarchy Ratios (Stable)
register_latency_ns = LATENCY_REGISTER_REF.m_as(NS)
hbm_vs_register = int(LATENCY_HBM3.m_as(NS) / register_latency_ns)
ssd_vs_register = int(round((LATENCY_NVME_SSD.m_as(NS) / register_latency_ns) / 100_000) * 100_000)
network_vs_local = int(LATENCY_INFINIBAND.m_as(NS) / LATENCY_HBM3.m_as(NS))
gpu_bw_vs_pcie = int(h_h100.memory.bandwidth.m_as(GB/second) / Hardware.Cloud.H100.interconnect.bandwidth.m_as(GB/second))
register_latency_ns = LATENCY_REGISTER_REF.to(NS).magnitude
hbm_vs_register = int(LATENCY_HBM3.to(NS).magnitude / register_latency_ns)
ssd_vs_register = int(round((LATENCY_NVME_SSD.to(NS).magnitude / register_latency_ns) / 100_000) * 100_000)
network_vs_local = int(LATENCY_INFINIBAND.to(NS).magnitude / LATENCY_HBM3.to(NS).magnitude)
gpu_bw_vs_pcie = int(h_h100.memory.bandwidth.to(GB/second).magnitude / Hardware.Cloud.H100.interconnect.bandwidth.to(GB/second).magnitude)
# Step 4: Current Hardware Reference (circa 2024)
lat_l1_ns = int(LATENCY_L1_REGISTER.m_as(NS))
lat_l2_ns = int(LATENCY_L2_CACHE.m_as(NS))
lat_hbm_ns = int(LATENCY_HBM3.m_as(NS))
lat_pcie_ns = int(LATENCY_PCIE_GEN5.m_as(NS))
lat_ib_ns = int(LATENCY_INFINIBAND.m_as(NS))
lat_ssd_ns = int(LATENCY_NVME_SSD.m_as(NS))
lat_l1_ns = int(LATENCY_L1_REGISTER.to(NS).magnitude)
lat_l2_ns = int(LATENCY_L2_CACHE.to(NS).magnitude)
lat_hbm_ns = int(LATENCY_HBM3.to(NS).magnitude)
lat_pcie_ns = int(LATENCY_PCIE_GEN5.to(NS).magnitude)
lat_ib_ns = int(LATENCY_INFINIBAND.to(NS).magnitude)
lat_ssd_ns = int(LATENCY_NVME_SSD.to(NS).magnitude)
bw_pcie5 = int(Hardware.Cloud.H100.interconnect.bandwidth.m_as(GB/second))
bw_dram = int(Hardware.Tech.Storage.SystemMemory.bandwidth.m_as(GB/second))
bw_ib = int(f.InfiniBand_NDR.bandwidth.m_as(GB/second))
bw_pcie5 = int(Hardware.Cloud.H100.interconnect.bandwidth.to(GB/second).magnitude)
bw_dram = int(Hardware.Tech.Storage.SystemMemory.bandwidth.to(GB/second).magnitude)
bw_ib = int(f.InfiniBand_NDR.bandwidth.to(GB/second).magnitude)
flops_h100_fp16 = int(h_h100.compute.peak_flops.m_as(TFLOPs/second))
flops_h100_fp8 = int(h_h100.compute.precision_flops["fp8"].m_as(TFLOPs/second))
flops_a100_fp16 = int(h_a100.compute.peak_flops.m_as(TFLOPs/second))
mobile_tops_int8 = int(Hardware.Mobile.iPhone15Pro.compute.peak_flops.m_as(TOPS))
flops_h100_fp16 = int(h_h100.compute.peak_flops.to(TFLOPs/second).magnitude)
flops_h100_fp8 = int(h_h100.compute.precision_flops["fp8"].to(TFLOPs/second).magnitude)
flops_a100_fp16 = int(h_a100.compute.peak_flops.to(TFLOPs/second).magnitude)
mobile_tops_int8 = int(Hardware.Mobile.iPhone15Pro.compute.peak_flops.to(TOPS).magnitude)
dc_mobile_ratio = flops_h100_fp16 / mobile_tops_int8
ridge_a100 = int(h_a100.ridge_point().m_as("flop/byte"))
ridge_h100 = int(h_h100.ridge_point().m_as("flop/byte"))
ridge_a100 = int(h_a100.ridge_point().to(flop/byte).magnitude)
ridge_h100 = int(h_h100.ridge_point().to(flop/byte).magnitude)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
# Canonical _str forms for prose; raw ints/floats above kept for arithmetic.
@@ -149,11 +149,11 @@ class NumbersToKnow:
lat_ib_ns_str = fmt(lat_ib_ns, precision=0, commas=False)
lat_ssd_ns_str = fmt(lat_ssd_ns, precision=0, commas=False)
bw_hbm_h100_str = fmt(h_h100.memory.bandwidth.m_as(TB/second), precision=1, commas=False)
bw_hbm_h100_str = fmt(h_h100.memory.bandwidth.to(TB/second).magnitude, precision=1, commas=False)
bw_pcie5_str = fmt(bw_pcie5, precision=0, commas=False)
bw_dram_str = fmt(bw_dram, precision=0, commas=False)
bw_ib_str = fmt(bw_ib, precision=0, commas=False)
bw_nvme_str = fmt(Hardware.Tech.Storage.NvmeGen4.bandwidth.m_as(GB/second), precision=0, commas=False)
bw_nvme_str = fmt(Hardware.Tech.Storage.NvmeGen4.bandwidth.to(GB/second).magnitude, precision=0, commas=False)
flops_h100_fp16_str = fmt(flops_h100_fp16, precision=0, commas=False)
flops_h100_fp8_str = fmt(flops_h100_fp8, precision=0, commas=False)
@@ -379,8 +379,8 @@ class A100RooflineExample:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
a100_fp16 = Hardware.Cloud.A100.compute.peak_flops
a100_bw = Hardware.Cloud.A100.memory.bandwidth
a100_fp16_raw_value = a100_fp16.m_as(TFLOPs/second)
a100_bw_raw_value = a100_bw.m_as(TB/second)
a100_fp16_raw_value = a100_fp16.to(TFLOPs/second).magnitude
a100_bw_raw_value = a100_bw.to(TB/second).magnitude
ridge_point_value = int(a100_fp16_raw_value / a100_bw_raw_value)
gemm_intensity_value = int(n_gemm_value / 3)
relu_achieved_tflops_value = relu_intensity_value * a100_bw_raw_value
@@ -652,7 +652,7 @@ class TrainingTimeRef:
d_tokens = Literature.Chinchilla.TokensPerParam * p_params
n_gpus = 1
x_flops_q = Hardware.Cloud.A100.compute.peak_flops
x_flops = x_flops_q.m_as(TFLOPs/second)
x_flops = x_flops_q.to(TFLOPs/second).magnitude
u_mfu = calibration.REFERENCE_MFU_SUSTAINED
# Rename variables for internal logic
@@ -663,7 +663,7 @@ class TrainingTimeRef:
u_mfu_value = u_mfu
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
total_flops = calc_transformer_training_flops(p_params_value, d_tokens_value).m_as(ureg.flop)
total_flops = calc_transformer_training_flops(p_params_value, d_tokens_value).to(flop).magnitude
# Step 1: x_flops_value is in TFLOP/s (1e12)
throughput = n_gpus_value * (x_flops_value) * u_mfu_value
@@ -761,9 +761,9 @@ class LittlesLawExample:
lambda_qps_value = 1000 # queries per second
w_latency_s_value = 0.050 # 50 ms in seconds
mem_per_req = 1 * GB
mem_per_req_gb_value = mem_per_req.m_as(GB)
mem_per_req_gb_value = mem_per_req.to(GB).magnitude
gpu_mem = 24 * GB
gpu_mem_gb_value = gpu_mem.m_as(GB)
gpu_mem_gb_value = gpu_mem.to(GB).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
l_concurrent_value = lambda_qps_value * w_latency_s_value
@@ -830,34 +830,34 @@ class ArchReferenceTables:
LATENCY_NVLINK = Hardware.Tech.Interconnect.NVLink.latency
LATENCY_PCIE_GEN5 = Hardware.Tech.Interconnect.PCIeGen5.latency
LATENCY_NVME_SSD = Hardware.Tech.Storage.NvmeGen4.latency
LATENCY_INFINIBAND = round(Systems.Fabrics.InfiniBand_NDR.latency.m_as(NS)) * NS # µs→ns, round to exact 5000
LATENCY_INFINIBAND = round(Systems.Fabrics.InfiniBand_NDR.latency.to(NS).magnitude) * NS # µs→ns, round to exact 5000
h100_l2 = Hardware.Cloud.H100.memory.l2_cache
tpuv5_l2 = Hardware.Cloud.TPUv5p.memory.l2_cache
register_ns_value = LATENCY_REGISTER_REF.m_as(NS)
register_ns_value = LATENCY_REGISTER_REF.to(NS).magnitude
analogy_base_seconds = 10
analogy_seconds_per_ns = analogy_base_seconds / register_ns_value
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
dram_pj_value = int(ENERGY_DRAM_ACCESS_PJ.m_as(ureg.picojoule))
flop_pj_value = ENERGY_FLOP_FP16_PJ.m_as(ureg.picojoule)
dram_pj_value = int(ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude)
flop_pj_value = ENERGY_FLOP_FP16_PJ.to(pJ).magnitude
energy_ratio_value = int(dram_pj_value / flop_pj_value)
l1_ns_value = int(LATENCY_L1_REGISTER.m_as(NS))
l2_ns_value = int(LATENCY_L2_CACHE.m_as(NS))
hbm_ns_value = int(LATENCY_HBM3.m_as(NS))
nvlink_ns_value = int(LATENCY_NVLINK.m_as(NS))
pcie_ns_value = int(LATENCY_PCIE_GEN5.m_as(NS))
ib_ns_value = int(LATENCY_INFINIBAND.m_as(NS))
ssd_ns_value = int(LATENCY_NVME_SSD.m_as(NS))
l1_ns_value = int(LATENCY_L1_REGISTER.to(NS).magnitude)
l2_ns_value = int(LATENCY_L2_CACHE.to(NS).magnitude)
hbm_ns_value = int(LATENCY_HBM3.to(NS).magnitude)
nvlink_ns_value = int(LATENCY_NVLINK.to(NS).magnitude)
pcie_ns_value = int(LATENCY_PCIE_GEN5.to(NS).magnitude)
ib_ns_value = int(LATENCY_INFINIBAND.to(NS).magnitude)
ssd_ns_value = int(LATENCY_NVME_SSD.to(NS).magnitude)
h100_flops = Hardware.Cloud.H100.compute.peak_flops
h100_bw = Hardware.Cloud.H100.memory.bandwidth
h100_cap = int(Hardware.Cloud.H100.memory.capacity.m_as(GB)) * GB
h100_cap = int(Hardware.Cloud.H100.memory.capacity.to(GB).magnitude) * GB
h100_nvlink = Hardware.Cloud.H100.nvlink.bandwidth
tpuv5_flops = Hardware.Cloud.TPUv5p.compute.peak_flops
tpuv5_bw = Hardware.Cloud.TPUv5p.memory.bandwidth
tpuv5_cap = int(Hardware.Cloud.TPUv5p.memory.capacity.m_as(GB)) * GB
tpuv5_cap = int(Hardware.Cloud.TPUv5p.memory.capacity.to(GB).magnitude) * GB
tpuv5_ici = Hardware.Cloud.TPUv5p.nvlink.bandwidth
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -902,17 +902,17 @@ class ArchReferenceTables:
tpuv5_ici_str = fmt_qty(tpuv5_ici, GB/second, precision=0, commas=False, unit_label="GB")
tpuv5_l2_mb_str = fmt_qty(tpuv5_l2, MB, precision=0, commas=False)
bw_hbm_str = fmt(Hardware.Cloud.H100.memory.bandwidth.m_as(GB/second), precision=0)
bw_nvlink_str = fmt(Hardware.Cloud.H100.nvlink.bandwidth.m_as(GB/second), precision=0)
bw_pcie_str = fmt(Hardware.Cloud.H100.interconnect.bandwidth.m_as(GB/second), precision=0)
bw_dram_str = fmt(Hardware.Tech.Storage.SystemMemory.bandwidth.m_as(GB/second), precision=0)
bw_ssd_str = fmt(Hardware.Tech.Storage.NvmeGen4.bandwidth.m_as(GB/second), precision=0, commas=False)
bw_net_str = fmt(Systems.Fabrics.InfiniBand_NDR.bandwidth.m_as(GB/second), precision=0, commas=False)
bw_hbm_str = fmt(Hardware.Cloud.H100.memory.bandwidth.to(GB/second).magnitude, precision=0)
bw_nvlink_str = fmt(Hardware.Cloud.H100.nvlink.bandwidth.to(GB/second).magnitude, precision=0)
bw_pcie_str = fmt(Hardware.Cloud.H100.interconnect.bandwidth.to(GB/second).magnitude, precision=0)
bw_dram_str = fmt(Hardware.Tech.Storage.SystemMemory.bandwidth.to(GB/second).magnitude, precision=0)
bw_ssd_str = fmt(Hardware.Tech.Storage.NvmeGen4.bandwidth.to(GB/second).magnitude, precision=0, commas=False)
bw_net_str = fmt(Systems.Fabrics.InfiniBand_NDR.bandwidth.to(GB/second).magnitude, precision=0, commas=False)
e_reg_str = MarkdownStr(f"{ENERGY_REG_PJ.m_as(ureg.picojoule):.2f}")
e_l1_str = MarkdownStr(f"{ENERGY_SRAM_L1_PJ.m_as(ureg.picojoule):.1f}")
e_l2_str = MarkdownStr(f"{ENERGY_SRAM_L2_PJ.m_as(ureg.picojoule):.1f}")
e_dram_str = MarkdownStr(f"{ENERGY_DRAM_ACCESS_PJ.m_as(ureg.picojoule):.0f}")
e_reg_str = MarkdownStr(f"{ENERGY_REG_PJ.to(pJ).magnitude:.2f}")
e_l1_str = MarkdownStr(f"{ENERGY_SRAM_L1_PJ.to(pJ).magnitude:.1f}")
e_l2_str = MarkdownStr(f"{ENERGY_SRAM_L2_PJ.to(pJ).magnitude:.1f}")
e_dram_str = MarkdownStr(f"{ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude:.0f}")
e_ssd_str = MarkdownStr("~5,000")
e_net_str = MarkdownStr("~10,000")
```
@@ -1065,10 +1065,10 @@ class BandwidthLatencySetup:
"""Namespace for bandwidth-latency lead-in constants."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
bw_gbps_value = Systems.Fabrics.Ethernet_10G.bandwidth.m_as(Gbps)
bw_gbps_value = Systems.Fabrics.Ethernet_10G.bandwidth.to(Gbps).magnitude
ping_ms_value = 10 # Network latency in ms
data = 1 * KB # Packet size
data_kb_value = data.m_as(KB)
data_kb_value = data.to(KB).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
# No derived values—only string formatting.
@@ -1108,8 +1108,8 @@ class BandwidthLatencyExample:
"""Namespace for latency-bound vs. bandwidth-bound transmission time example."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
data_kb_value = (1 * KB).m_as(byte)
bw_gbps_value = Systems.Fabrics.Ethernet_10G.bandwidth.m_as(Gbps) * BILLION
data_kb_value = (1 * KB).to(byte).magnitude
bw_gbps_value = Systems.Fabrics.Ethernet_10G.bandwidth.to(Gbps).magnitude * BILLION
large_data_gb_value = 1
ping_ms_value = 10
@@ -1117,7 +1117,7 @@ class BandwidthLatencyExample:
tx_time_s_value = (data_kb_value * BITS_PER_BYTE) / bw_gbps_value
tx_time_us_value = tx_time_s_value * MILLION
large_data_bits_value = large_data_gb_value * (1 * GB).m_as(byte) * BITS_PER_BYTE
large_data_bits_value = large_data_gb_value * (1 * GB).to(byte).magnitude * BITS_PER_BYTE
large_tx_time_s_value = large_data_bits_value/bw_gbps_value
total_large_time_ms_value = ping_ms_value + large_tx_time_s_value * THOUSAND
book/quarto/contents/vol1/benchmarking/benchmarking.qmd
+62 -62
View File
@@ -67,7 +67,7 @@ from mlsysim.core.constants import TFLOPs, second
from mlsysim.fmt import fmt, fmt_qty
class A100PeakBenchmark:
_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs / second)
_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs / second).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
a100_tflops_fp16_str = fmt_qty(_peak_tflops * TFLOP / second, TFLOP / second, precision=0, commas=False)
@@ -114,8 +114,8 @@ class MobileNetLighthouseSetup:
_mobilenet_v2 = Models.Vision.MobileNetV2
_edgetpu_latency_ms = 2
_cpu_latency_ms = 15
_mv2_size_mb = _mobilenet_v2.size_in_bytes(BYTES_FP32).m_as(MB)
_mv2_int8_mb = _mobilenet_v2.size_in_bytes(BYTES_INT8).m_as(MB)
_mv2_size_mb = _mobilenet_v2.size_in_bytes(BYTES_FP32).to(MB).magnitude
_mv2_int8_mb = _mobilenet_v2.size_in_bytes(BYTES_INT8).to(MB).magnitude
_mv2_compress = _mv2_size_mb / _mv2_int8_mb
check(_mv2_compress == 4.0, f"Expected 4x INT8 compression, got {_mv2_compress:.1f}x")
@@ -275,8 +275,8 @@ from mlsysim.fmt import fmt, fmt_qty
class MobileNetEnergyFlops:
_flops_ratio = (
Models.Vision.ResNet50.inference_flops.m_as(GFLOPs)
/ Models.Vision.MobileNetV2.inference_flops.m_as(GFLOPs)
Models.Vision.ResNet50.inference_flops.to(GFLOPs).magnitude
/ Models.Vision.MobileNetV2.inference_flops.to(GFLOPs).magnitude
)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -602,12 +602,12 @@ System benchmarks serve two functions. For practitioners, they enable informed h
class A100Roofline:
"""A100 hardware constants for roofline analysis."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
_a100_fp16 = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
_a100_fp32 = Hardware.Cloud.A100.compute.precision_flops["fp32"].m_as(TFLOPs/second)
_a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
_a100_fp16 = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
_a100_fp32 = Hardware.Cloud.A100.compute.precision_flops["fp32"].to(TFLOPs/second).magnitude
_a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
_a100_ridge = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
_a100_ridge = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
a100_tflops_fp16_str = fmt_qty(_a100_fp16 * TFLOP / second, TFLOP / second, precision=0, commas=False)
@@ -661,8 +661,8 @@ class RooflineExamples:
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
# A100 Specs (re-derived locally for safety)
peak_flops = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
peak_bw = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
peak_flops = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
peak_bw = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
ridge_point = peak_flops / peak_bw # ~153
# ResNet (Compute Bound)
@@ -672,8 +672,8 @@ class RooflineExamples:
# BERT (Memory Bound at Batch=1)
bert_model = Models.Language.BERT_Base
bert_flops_b = bert_model.inference_flops.m_as(Bparam)
bert_weight_mb = bert_model.size_in_bytes(BYTES_FP32).m_as(MB)
bert_flops_b = bert_model.inference_flops.to(Bparam).magnitude
bert_weight_mb = bert_model.size_in_bytes(BYTES_FP32).to(MB).magnitude
bert_util_peak = 0.85
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
@@ -779,25 +779,25 @@ class BertRoofline:
p_b1 = Engine.solve(m_bert, h_a100, batch_size=batch_1, precision="fp32", efficiency=1.0)
p_b32 = Engine.solve(m_bert, h_a100, batch_size=batch_32, precision="fp32", efficiency=0.85)
bert_params_m = m_bert.parameters.m_as(Mparam)
bert_flops_b = m_bert.inference_flops.m_as(Bparam)
bert_weight_mb = p_b1.memory_footprint.m_as(MB)
bert_params_m = m_bert.parameters.to(Mparam).magnitude
bert_flops_b = m_bert.inference_flops.to(Bparam).magnitude
bert_weight_mb = p_b1.memory_footprint.to(MB).magnitude
ai_b1 = (m_bert.inference_flops / p_b1.memory_footprint).m_as("flop/byte")
ai_b1 = (m_bert.inference_flops / p_b1.memory_footprint).to(flop/byte).magnitude
# Realized TFLOP/s = throughput * flops_per_inference
perf_b1 = (p_b1.throughput * m_bert.inference_flops).m_as(TFLOPs/second)
perf_b1 = (p_b1.throughput * m_bert.inference_flops).to(TFLOPs/second).magnitude
util_b1 = p_b1.mfu * 100.0
flops_b32 = bert_flops_b * batch_32
ai_b32 = (m_bert.inference_flops * batch_32 / p_b1.memory_footprint).m_as("flop/byte")
ai_b32 = (m_bert.inference_flops * batch_32 / p_b1.memory_footprint).to(flop/byte).magnitude
# Is it compute bound now?
is_compute_bound_b32 = p_b32.bottleneck == "Compute"
perf_b32 = (p_b32.throughput * m_bert.inference_flops).m_as(TFLOPs/second)
perf_b32 = (p_b32.throughput * m_bert.inference_flops).to(TFLOPs/second).magnitude
peak_flops = h_a100.compute.peak_flops.m_as(TFLOPs/second)
peak_bw = h_a100.memory.bandwidth.m_as(TB/second)
ridge_point = h_a100.ridge_point().m_as("flop/byte")
peak_flops = h_a100.compute.peak_flops.to(TFLOPs/second).magnitude
peak_bw = h_a100.memory.bandwidth.to(TB/second).magnitude
ridge_point = h_a100.ridge_point().to(flop/byte).magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(ai_b32 > ai_b1, "Batching must increase Arithmetic Intensity.")
@@ -1126,8 +1126,8 @@ from mlsysim.fmt import fmt
class H100SolCheck:
"""H100 peak throughput for micro-benchmark SOL checks."""
_h100_fp16 = Hardware.Cloud.H100.compute.peak_flops.m_as(TFLOPs / second)
_h100_fp8 = Hardware.Cloud.H100.compute.precision_flops["fp8"].m_as(TFLOPs / second)
_h100_fp16 = Hardware.Cloud.H100.compute.peak_flops.to(TFLOPs / second).magnitude
_h100_fp8 = Hardware.Cloud.H100.compute.precision_flops["fp8"].to(TFLOPs / second).magnitude
sol_example_tflops_value = 10
cache_bw_low_tbs_value = 5
cache_bw_high_tbs_value = 10
@@ -1522,7 +1522,7 @@ Dataset selection shapes everything downstream. In the audio anomaly detection e
class BertModelSpecs:
"""BERT-Large parameter count for the model selection footnote."""
_bert_large_m = Models.Language.BERT_Large.parameters.m_as(Mparam)
_bert_large_m = Models.Language.BERT_Large.parameters.to(Mparam).magnitude
bert_large_params_m_str = fmt(_bert_large_m, precision=0, commas=False)
```
@@ -1616,10 +1616,10 @@ class AnomalySpecs:
"""Anomaly detection model constants for benchmark component examples."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
from mlsysim import Scenarios
_anomaly_k = Models.Tiny.AnomalyDetector.parameters.m_as(Kparam)
_anomaly_latency_ms = Scenarios.AnomalyModel.Latency.m_as(ureg.ms)
_anomaly_k = Models.Tiny.AnomalyDetector.parameters.to(Kparam).magnitude
_anomaly_latency_ms = Scenarios.AnomalyModel.Latency.to(ms).magnitude
_anomaly_auc = Scenarios.AnomalyModel.Auc
_anomaly_energy_uj = Scenarios.AnomalyModel.Energy.m_as(ureg.microjoule)
_anomaly_energy_uj = Scenarios.AnomalyModel.Energy.to(ureg.microjoule).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
anomaly_params_k_str = fmt_count(_anomaly_k * THOUSAND, scale='K', precision=0, commas=False)
@@ -1705,8 +1705,8 @@ class MobileNetTradeoffCalc:
latency_int8_ms = 35
acc_fp32 = 71.8
acc_int8 = 70.9
size_fp32_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_FP32).m_as(MB)
size_int8_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_INT8).m_as(MB)
size_fp32_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_FP32).to(MB).magnitude
size_int8_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_INT8).to(MB).magnitude
milliseconds_per_second = 1000
# EXECUTE
@@ -1829,8 +1829,8 @@ from mlsysim.fmt import fmt_count
class CompressionModelSpecs:
"""MobileNetV2 vs ResNet-50 parameter counts for compression benchmarks."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
_mobilenet_m = Models.Vision.MobileNetV2.parameters.m_as(Mparam)
_resnet50_m = Models.Vision.ResNet50.parameters.m_as(Mparam)
_mobilenet_m = Models.Vision.MobileNetV2.parameters.to(Mparam).magnitude
_resnet50_m = Models.Vision.ResNet50.parameters.to(Mparam).magnitude
_mobilenet_top1 = 72
_resnet50_top1 = 76
@@ -1970,16 +1970,16 @@ from mlsysim.core.units import MWh
class InferenceEnergy:
"""Per-query energy calculation for two inference latency scenarios."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
_power_w = Hardware.Cloud.V100.tdp.m_as(watt) # Typical accelerator TDP (watts)
_power_w = Hardware.Cloud.V100.tdp.to(watt).magnitude # Typical accelerator TDP (watts)
_latency_fast_ms = 10 # Fast inference (ms)
_latency_slow_ms = 100 # Slow inference (ms)
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
_latency_fast_s = (_latency_fast_ms * ms).m_as(second) # seconds
_latency_fast_s = (_latency_fast_ms * ms).to(second).magnitude # seconds
_energy_fast_j = _power_w * _latency_fast_s # joules
_energy_fast_wh = _energy_fast_j / (1 * hour).m_as(second) # watt-hours
_energy_slow_j = _power_w * (_latency_slow_ms * ms).m_as(second)
_energy_slow_wh = _energy_slow_j / (1 * hour).m_as(second)
_energy_fast_wh = _energy_fast_j / (1 * hour).to(second).magnitude # watt-hours
_energy_slow_j = _power_w * (_latency_slow_ms * ms).to(second).magnitude
_energy_slow_wh = _energy_slow_j / (1 * hour).to(second).magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(_energy_fast_j == 3.0, f"Expected 3 J at 300 W for 10 ms, got {_energy_fast_j:.1f}")
@@ -2040,7 +2040,7 @@ These benchmarks are vital because training represents the largest capital expen
class GPT3TrainingSpecs:
"""GPT-3 parameter and token counts for training benchmark context."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
_params_b = Models.Language.GPT3.parameters.m_as(Bparam)
_params_b = Models.Language.GPT3.parameters.to(Bparam).magnitude
_tokens_b = Models.Language.GPT3.training_tokens.to('count').magnitude / BILLION
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -2452,11 +2452,11 @@ class EnergyPerOp:
# FP constants use pJ/FLOP; INT8 uses pJ/op. Use .magnitude for bare numeric values.
_fp32_pj = ENERGY_FLOP_FP32_PJ.magnitude
_fp16_pj = ENERGY_FLOP_FP16_PJ.magnitude
_int8_pj = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count)
_int8_pj = ENERGY_OP_INT8_PJ.to(pJ).magnitude
_reg_pj = ENERGY_REG_PJ
_l1_pj = ENERGY_SRAM_L1_PJ
_l2_pj = ENERGY_SRAM_L2_PJ
_dram_pj = ENERGY_DRAM_PJ_PER_BYTE.m_as(ureg.picojoule/ureg.byte)
_dram_pj = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude
dram_vs_reg_ratio_value = round(_dram_pj / _reg_pj / 1000) * THOUSAND
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -2484,30 +2484,30 @@ from mlsysim.fmt import fmt, check
class EnergyBreakdownCalc:
"""FP32 vs INT8 MobileNet inference energy: memory load dominates; INT8 attacks both sources."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
m_params = Models.Vision.MobileNetV2.parameters.m_as(Mparam)
m_ops = Models.Vision.MobileNetV2.inference_flops.m_as(MFLOPs)
m_params = Models.Vision.MobileNetV2.parameters.to(Mparam).magnitude
m_ops = Models.Vision.MobileNetV2.inference_flops.to(MFLOPs).magnitude
ENERGY_DRAM_PJ_PER_BYTE = Hardware.Tech.Memory.DRAM.energy_per_byte
ENERGY_FLOP_FP32_PJ = Hardware.Tech.Op.FlopFp32.energy
ENERGY_OP_INT8_PJ = Hardware.Tech.Op.OpInt8.energy
_dram_pj = ENERGY_DRAM_PJ_PER_BYTE.m_as(ureg.picojoule/byte)
_fp32_pj = ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule/ureg.flop)
_int8_pj = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count)
_bytes_fp32 = BYTES_FP32.m_as(byte)
_bytes_int8 = BYTES_INT8.m_as(byte)
_dram_pj = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude
_fp32_pj = ENERGY_FLOP_FP32_PJ.to(pJ / flop).magnitude
_int8_pj = ENERGY_OP_INT8_PJ.to(pJ).magnitude
_bytes_fp32 = BYTES_FP32.to(byte).magnitude
_bytes_int8 = BYTES_INT8.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
e_fp32_load = (m_params * MILLION * _bytes_fp32 * _dram_pj) / MILLION
e_fp32_compute = (m_ops * MILLION * _fp32_pj) / MILLION
e_int8_load = (m_params * MILLION * _bytes_int8 * _dram_pj) / MILLION
e_int8_compute = (m_ops * MILLION * _int8_pj) / MILLION
m_fp32_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_FP32).m_as(MB)
m_int8_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_INT8).m_as(MB)
m_fp32_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_FP32).to(MB).magnitude
m_int8_mb = Models.Vision.MobileNetV2.size_in_bytes(BYTES_INT8).to(MB).magnitude
e_fp32_total = e_fp32_load + e_fp32_compute
e_int8_total = e_int8_load + e_int8_compute
s_load = e_fp32_load/e_int8_load
s_compute = e_fp32_compute/e_int8_compute
s_total = e_fp32_total/e_int8_total
e_fp32_load_mj = (e_fp32_load * ureg.microjoule).m_as(ureg.millijoule)
e_fp32_compute_mj = (e_fp32_compute * ureg.microjoule).m_as(ureg.millijoule)
e_fp32_load_mj = (e_fp32_load * ureg.microjoule).to(mJ).magnitude
e_fp32_compute_mj = (e_fp32_compute * ureg.microjoule).to(mJ).magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(round(m_fp32_mb) == 14, f"MobileNetV2 FP32 size should be about 14 MB, got {m_fp32_mb:.1f}")
check(s_load == 4.0, f"Expected 4x load-energy reduction, got {s_load:.1f}x")
@@ -3069,7 +3069,7 @@ from mlsysim.fmt import fmt, fmt_qty, check, fmt_count
class ColdStartTransferCalc:
"""FP16 weight transfer lower bound for a 7-billion-parameter model on PCIe 4.0."""
# LOAD
params_b = Models.Language.Llama2_7B.parameters.m_as(Bparam)
params_b = Models.Language.Llama2_7B.parameters.to(Bparam).magnitude
bytes_per_param = 2
pcie_effective_gbs = 25
@@ -3117,7 +3117,7 @@ class TopsFootnoteAnchor:
"""Local H100 INT8 TOPS anchor for the metric-comparability footnote."""
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
h100_tops_int8 = Hardware.Cloud.H100.compute.precision_flops["int8"].m_as(TOPS)
h100_tops_int8 = Hardware.Cloud.H100.compute.precision_flops["int8"].to(TOPS).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
# Direct hardware-constant lookup.
@@ -4514,9 +4514,9 @@ class MobileNetINT8Calc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
_mobilenet_v2 = Models.Vision.MobileNetV2
acc_fp32 = 71.8
params_m = _mobilenet_v2.parameters.m_as(Mparam)
size_fp32_mb = _mobilenet_v2.size_in_bytes(BYTES_FP32).m_as(MB)
size_int8_mb = _mobilenet_v2.size_in_bytes(BYTES_INT8).m_as(MB)
params_m = _mobilenet_v2.parameters.to(Mparam).magnitude
size_fp32_mb = _mobilenet_v2.size_in_bytes(BYTES_FP32).to(MB).magnitude
size_int8_mb = _mobilenet_v2.size_in_bytes(BYTES_INT8).to(MB).magnitude
acc_int8 = 70.9
top5_fp32 = 91.0
top5_int8 = 90.4
@@ -5254,14 +5254,14 @@ class PrecisionEnergySummary:
"""Local summary ratio for MobileNetV2 FP32 vs. INT8 inference energy."""
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
m_params = Models.Vision.MobileNetV2.parameters.m_as(Mparam)
m_ops = Models.Vision.MobileNetV2.inference_flops.m_as(MFLOPs)
m_params = Models.Vision.MobileNetV2.parameters.to(Mparam).magnitude
m_ops = Models.Vision.MobileNetV2.inference_flops.to(MFLOPs).magnitude
ENERGY_DRAM_PJ_PER_BYTE = Hardware.Tech.Memory.DRAM.energy_per_byte
ENERGY_FLOP_FP32_PJ = Hardware.Tech.Op.FlopFp32.energy
ENERGY_OP_INT8_PJ = Hardware.Tech.Op.OpInt8.energy
dram_pj = ENERGY_DRAM_PJ_PER_BYTE.m_as(ureg.picojoule/byte)
fp32_pj = ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule/ureg.flop)
int8_pj = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count)
dram_pj = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude
fp32_pj = ENERGY_FLOP_FP32_PJ.to(pJ / flop).magnitude
int8_pj = ENERGY_OP_INT8_PJ.to(pJ).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
e_fp32_load = (m_params * MILLION * 4 * dram_pj) / MILLION
book/quarto/contents/vol1/conclusion/conclusion.qmd
+25 -25
View File
@@ -152,14 +152,14 @@ class MobileNetJourneyMath:
# GUARD
check(8 <= standard_to_depthwise <= 9, "Depthwise separable example should land near 8--9x.")
check(13 <= resnet_to_mobilenet.m_as("") <= 15, "ResNet-50/MobileNetV2 ratio drifted.")
check(int8_vs_fp32.m_as("") == 4 and int8_vs_fp16.m_as("") == 2, "INT8 byte ratios changed.")
check(13 <= resnet_to_mobilenet.to("").magnitude <= 15, "ResNet-50/MobileNetV2 ratio drifted.")
check(int8_vs_fp32.to("").magnitude == 4 and int8_vs_fp16.to("").magnitude == 2, "INT8 byte ratios changed.")
# OUTPUT
depthwise_reduction_str = fmt_multiple(standard_to_depthwise, precision=1, commas=False)
resnet_ops_reduction_str = fmt_multiple(resnet_to_mobilenet.m_as(""), precision=1, commas=False)
int8_fp32_reduction_str = fmt_multiple(int8_vs_fp32.m_as(""), precision=0, commas=False)
int8_fp16_reduction_str = fmt_multiple(int8_vs_fp16.m_as(""), precision=0, commas=False)
resnet_ops_reduction_str = fmt_multiple(resnet_to_mobilenet.to("").magnitude, precision=1, commas=False)
int8_fp32_reduction_str = fmt_multiple(int8_vs_fp32.to("").magnitude, precision=0, commas=False)
int8_fp16_reduction_str = fmt_multiple(int8_vs_fp16.to("").magnitude, precision=0, commas=False)
```
@Tbl-lighthouse-journey-mobilenet traces this journey for a single model, MobileNetV2, demonstrating how every chapter's principles converge on a single engineering artifact. The table walks through seven phases (from foundational constraints through architecture, training, compression, acceleration, serving, and operations) showing how each phase's decisions propagate forward to shape what becomes possible in subsequent phases.
@@ -223,12 +223,12 @@ class EnergyMovementMath:
dram_vs_fp16 = dram_access / fp16_flop
# GUARD
check(100 <= dram_vs_fp32.m_as("") <= 1000, "DRAM/FP32 energy ratio left expected range.")
check(100 <= dram_vs_fp16.m_as("") <= 1000, "DRAM/FP16 energy ratio left expected range.")
check(100 <= dram_vs_fp32.to("").magnitude <= 1000, "DRAM/FP32 energy ratio left expected range.")
check(100 <= dram_vs_fp16.to("").magnitude <= 1000, "DRAM/FP16 energy ratio left expected range.")
# OUTPUT
dram_vs_fp32_str = fmt_int(round(dram_vs_fp32.m_as("")), commas=False)
dram_vs_fp16_str = fmt_multiple(dram_vs_fp16.m_as(""), precision=1, commas=False)
dram_vs_fp32_str = fmt_int(round(dram_vs_fp32.to("").magnitude), commas=False)
dram_vs_fp16_str = fmt_multiple(dram_vs_fp16.to("").magnitude, precision=1, commas=False)
```
| **#** | **Principle** | **Part** | **Core Equation/Statement** | **What It Predicts** |
@@ -671,15 +671,15 @@ class ConclusionRoofline:
t_mem_ms_str = fmt_qty(t_mem, ms, precision=1, commas=False)
t_comp_ms_str = fmt_qty(t_comp, ms, precision=2, commas=False)
ratio_str = fmt_multiple(ratio.m_as(''), precision=1, commas=False)
ratio_str = fmt_multiple(ratio.to("").magnitude, precision=1, commas=False)
# LaTeX equations — use raw magnitudes, not suffixed _str exports
dvol_gb_val = d_vol.m_as('GB')
h100_bw_gb_val = gpu.memory.bandwidth.m_as('GB/s')
compute_gflops_val = compute_req.m_as('GFLOPs')
h100_peak_tflops_val = gpu.compute.peak_flops.m_as('TFLOP/s')
t_mem_ms_val = t_mem.m_as('ms')
t_comp_ms_val = t_comp.m_as('ms')
dvol_gb_val = d_vol.to(GB).magnitude
h100_bw_gb_val = gpu.memory.bandwidth.to(GB / second).magnitude
compute_gflops_val = compute_req.to(GFLOPs).magnitude
h100_peak_tflops_val = gpu.compute.peak_flops.to(TFLOP / second).magnitude
t_mem_ms_val = t_mem.to(ms).magnitude
t_comp_ms_val = t_comp.to(ms).magnitude
t_mem_eq = fmt_math(f"T_{{\\text{{mem}}}} = \\frac{{{dvol_gb_val:.0f} \\text{{ GB}}}}{{{h100_bw_gb_val:.0f} \\text{{ GB/s}}}} \\approx {t_mem_ms_val:.1f} \\text{{ ms}}")
t_comp_eq = fmt_math(f"T_{{\\text{{comp}}}} = \\frac{{{compute_gflops_val:.0f} \\times 10^9}}{{{h100_peak_tflops_val:.0f} \\times 10^{{12}}}} = {t_comp_ms_val:.2f} \\text{{ ms}}")
@@ -833,14 +833,14 @@ class EdgeReferenceRatios:
memory_ratio = h_h100.memory.capacity / phone_ram
# GUARD
check(int8_peak_ratio.m_as("") > 10, "Mobile edge example should be far below H100 throughput.")
check(memory_ratio.m_as("") > 5, "Mobile edge example should have meaningfully less memory headroom.")
check(power_ratio.m_as("") > 100, "Mobile edge example should have a much smaller power envelope.")
check(int8_peak_ratio.to("").magnitude > 10, "Mobile edge example should be far below H100 throughput.")
check(memory_ratio.to("").magnitude > 5, "Mobile edge example should have meaningfully less memory headroom.")
check(power_ratio.to("").magnitude > 100, "Mobile edge example should have a much smaller power envelope.")
# OUTPUT
int8_peak_ratio_str = fmt_multiple(int8_peak_ratio.m_as(""), precision=1, commas=False)
power_ratio_str = fmt_multiple(power_ratio.m_as(""), precision=1, commas=False)
memory_ratio_str = fmt_multiple(memory_ratio.m_as(""), precision=1, commas=False)
int8_peak_ratio_str = fmt_multiple(int8_peak_ratio.to("").magnitude, precision=1, commas=False)
power_ratio_str = fmt_multiple(power_ratio.to("").magnitude, precision=1, commas=False)
memory_ratio_str = fmt_multiple(memory_ratio.to("").magnitude, precision=1, commas=False)
```
\index{AI Democratization!edge deployment}
@@ -946,17 +946,17 @@ class FleetFailureMath:
gpu_mtbf = float(Systems.Reliability.Gpu.mttf_hours) * ureg.hour
# EXECUTE
gpu_mtbf_years = gpu_mtbf.m_as("hour") / (HOURS_PER_DAY * DAYS_PER_YEAR)
gpu_mtbf_years = gpu_mtbf.to(hour).magnitude / (HOURS_PER_DAY * DAYS_PER_YEAR)
cluster_mtbf = calc_mtbf_cluster(gpu_mtbf, gpu_count)
# GUARD
check(gpu_mtbf_years > 1, "Component MTTF should read as years.")
check(cluster_mtbf.m_as("hour") < 100, "Thousand-GPU MTBF should read as hours.")
check(cluster_mtbf.to(hour).magnitude < 100, "Thousand-GPU MTBF should read as hours.")
# OUTPUT
gpu_count_str = fmt(gpu_count, precision=0)
gpu_mtbf_years_str = fmt_time(gpu_mtbf_years, 'year', precision=1, commas=False, style='word')
cluster_mtbf_hours_str = fmt_time(cluster_mtbf.m_as("hour"), 'hour', precision=1, commas=False, style='word')
cluster_mtbf_hours_str = fmt_time(cluster_mtbf.to(hour).magnitude, 'hour', precision=1, commas=False, style='word')
```
### Node to fleet { .unnumbered}
book/quarto/contents/vol1/hw_acceleration/hw_acceleration.qmd
+117 -117
View File
@@ -56,8 +56,8 @@ from mlsysim.fmt import fmt_int, fmt, fmt_qty, fmt_qty_int, check
class HwAccelSetup:
"""A100 peak FP16/BF16 throughput vs. server CPU for the opening callout."""
# LOAD
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
cpu_tflops_low = Hardware.Cloud.ReferenceCPU.compute.peak_flops.m_as(TFLOPs/second)
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
cpu_tflops_low = Hardware.Cloud.ReferenceCPU.compute.peak_flops.to(TFLOPs/second).magnitude
cpu_tflops_high = 2
# EXECUTE
@@ -276,7 +276,7 @@ class AmdahlH100:
# ┌── 4. OUTPUT (Formatting) ──────────────────────────────────────────────
# Hardware context
h100_tops_int8 = Hardware.Cloud.H100.compute.precision_flops["int8"].m_as(TOPS)
h100_tops_int8 = Hardware.Cloud.H100.compute.precision_flops["int8"].to(TOPS).magnitude
h100_tops_int8_str = fmt(h100_tops_int8, precision=0)
hw_speedup_str = fmt(hw_speedup_factor, precision=0, commas=False)
@@ -795,10 +795,10 @@ class CpuMlInefficiency:
cpu_gflops_str = fmt(cpu_gflops_value, precision=0, commas=False)
# A100/Mobile specs for footnote comparison
a100_tflops_fp16 = f"{Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second):.0f}"
_tf32_tflops = Hardware.Cloud.A100.compute.precision_flops['tf32'].m_as(TFLOPs/second)
a100_tflops_fp16 = f"{Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude:.0f}"
_tf32_tflops = Hardware.Cloud.A100.compute.precision_flops['tf32'].to(TFLOPs/second).magnitude
a100_tflops_tf32 = f"{_tf32_tflops:.0f}"
mobile_tops = f"{Hardware.Mobile.iPhone15Pro.compute.peak_flops.m_as(TOPS):.0f}"
mobile_tops = f"{Hardware.Mobile.iPhone15Pro.compute.peak_flops.to(TOPS).magnitude:.0f}"
```
### Machine learning hardware specialization {#sec-hardware-acceleration-machine-learning-hardware-specialization-09c5}
@@ -815,9 +815,9 @@ class MlAcceleratorCallout:
"""A100 throughput and bandwidth vs. server CPU for the ML accelerator callout."""
# LOAD
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
cpu_tflops_low = Hardware.Cloud.ReferenceCPU.compute.peak_flops.m_as(TFLOPs/second)
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
cpu_tflops_low = Hardware.Cloud.ReferenceCPU.compute.peak_flops.to(TFLOPs/second).magnitude
cpu_tflops_high = 2
# EXECUTE
@@ -1993,7 +1993,7 @@ class SystolicEnergy:
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
# Energy value sourced from the mlsysim registry (Hardware.Tech.Memory).
ENERGY_DRAM_ACCESS_PJ = Hardware.Tech.Memory.DRAM.energy_per_access
dram_pj = ENERGY_DRAM_ACCESS_PJ.m_as('pJ')
dram_pj = ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude
mac_pj = 1.0 # Compute cost
# Vector Unit: Needs 3 loads (A, B, C) + 1 write (C) per MAC
@@ -2324,7 +2324,7 @@ class SystolicOpsCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
SYSTOLIC_ARRAY_DIM = 128 # pedagogical systolic-array dimension example
systolic_dim_value = SYSTOLIC_ARRAY_DIM
tpuv4_bw_gbs = Hardware.Cloud.TPUv4.memory.bandwidth.m_as(GB/second)
tpuv4_bw_gbs = Hardware.Cloud.TPUv4.memory.bandwidth.to(GB/second).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
systolic_ops_value = systolic_dim_value * systolic_dim_value
@@ -2415,10 +2415,10 @@ class InterconnectBandwidth:
"""Namespace for Interconnect Hierarchy bandwidth taper."""
h_h100 = Hardware.Cloud.H100
hbm_bw = h_h100.memory.bandwidth.m_as(GB/second)
nvlink_bw = Hardware.Cloud.H100.nvlink.bandwidth.m_as(GB/second)
pcie_bw = Hardware.Cloud.H100.interconnect.bandwidth.m_as(GB/second)
net_bw = Systems.Fabrics.InfiniBand_NDR.bandwidth.m_as(GB/second)
hbm_bw = h_h100.memory.bandwidth.to(GB/second).magnitude
nvlink_bw = Hardware.Cloud.H100.nvlink.bandwidth.to(GB/second).magnitude
pcie_bw = Hardware.Cloud.H100.interconnect.bandwidth.to(GB/second).magnitude
net_bw = Systems.Fabrics.InfiniBand_NDR.bandwidth.to(GB/second).magnitude
# Invariant: HBM > NVLink > PCIe > Network
check(hbm_bw > nvlink_bw > pcie_bw > net_bw, "Bandwidth taper violated!")
@@ -2510,27 +2510,27 @@ class AcceleratorEconomics:
h_h100 = Hardware.Cloud.H100
h_tpu = Hardware.Cloud.TPUv4
price_v100 = Hardware.Cloud.V100.unit_cost.m_as("dollar") # older generation
price_a100 = Hardware.Cloud.A100.unit_cost.m_as("dollar") # current workhorse
price_v100 = Hardware.Cloud.V100.unit_cost.to(USD).magnitude # older generation
price_a100 = Hardware.Cloud.A100.unit_cost.to(USD).magnitude # current workhorse
price_h100 = 25000 # lower bound of range
price_tpu = 8000 # estimated from cloud rates
price_gaudi = 12000 # Intel alternative
# Gaudi 2 (FP8 — Gaudi 2's flagship precision)
gaudi_tf = Hardware.Cloud.Gaudi2.compute.precision_flops["fp8"].m_as(TFLOPs/second)
gaudi_bw_value = Hardware.Cloud.Gaudi2.memory.bandwidth.m_as(GB/second)
gaudi_tf = Hardware.Cloud.Gaudi2.compute.precision_flops["fp8"].to(TFLOPs/second).magnitude
gaudi_bw_value = Hardware.Cloud.Gaudi2.memory.bandwidth.to(GB/second).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
v100_tf = h_v100.compute.peak_flops.m_as(TFLOPs/second)
v100_tf = h_v100.compute.peak_flops.to(TFLOPs/second).magnitude
v100_ratio = price_v100 / v100_tf
a100_tf = h_a100.compute.peak_flops.m_as(TFLOPs/second)
a100_tf = h_a100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_ratio = price_a100 / a100_tf
h100_tf = h_h100.compute.precision_flops["tf32"].m_as(TFLOPs/second)
h100_tf = h_h100.compute.precision_flops["tf32"].to(TFLOPs/second).magnitude
h100_ratio = price_h100 / h100_tf
tpu_tf = h_tpu.compute.peak_flops.m_as(TFLOPs/second)
tpu_tf = h_tpu.compute.peak_flops.to(TFLOPs/second).magnitude
tpu_ratio = price_tpu/tpu_tf
gaudi_ratio = price_gaudi/gaudi_tf
@@ -2552,8 +2552,8 @@ class AcceleratorEconomics:
# H100 specs
h100_tflops_tf32_str = fmt_qty(h_h100.compute.precision_flops["tf32"], TFLOP/second, precision=0, commas=False)
h100_tflops_fp16_str = fmt(h_h100.compute.peak_flops.m_as(TFLOPs/second), precision=0, commas=False)
h100_tflops_fp8_str = fmt(Hardware.Cloud.H100.compute.precision_flops["fp8"].m_as(TFLOPs/second), precision=0)
h100_tflops_fp16_str = fmt(h_h100.compute.peak_flops.to(TFLOPs/second).magnitude, precision=0, commas=False)
h100_tflops_fp8_str = fmt(Hardware.Cloud.H100.compute.precision_flops["fp8"].to(TFLOPs/second).magnitude, precision=0)
h100_bw_str = fmt_qty(h_h100.memory.bandwidth, GB/second, precision=0, commas=True)
h100_bw_tbs_str = fmt_qty(h_h100.memory.bandwidth, TB/second, precision=2, commas=False)
h100_price_str = fmt_usd_range(price_h100, 30000, precision=0, approx=True, repeat_symbol=False)
@@ -2607,7 +2607,7 @@ from mlsysim.fmt import fmt
class B200FP4Specs:
"""B200 FP4 peak throughput (dense and sparse)."""
dense_pflops = Hardware.Cloud.B200.compute.precision_flops["fp4"].m_as(PFLOPs/second)
dense_pflops = Hardware.Cloud.B200.compute.precision_flops["fp4"].to(PFLOPs/second).magnitude
sparse_pflops = 2 * dense_pflops
dense_pflops_str = fmt_qty(dense_pflops * PFLOPs / second, PFLOPs / second, precision=0, commas=False, unit_label="PFLOP/s")
sparse_pflops_str = fmt_qty(sparse_pflops * PFLOPs / second, PFLOPs / second, precision=0, commas=False, unit_label="PFLOP/s")
@@ -2722,8 +2722,8 @@ from mlsysim.fmt import fmt
class MemoryWallH100Specs:
"""H100 peak throughput and bandwidth for the memory-wall scaling prose."""
h100_tflops_fp16 = Hardware.Cloud.H100.compute.peak_flops.m_as(TFLOPs/second)
h100_bw_tbs = Hardware.Cloud.H100.memory.bandwidth.m_as(TB/second)
h100_tflops_fp16 = Hardware.Cloud.H100.compute.peak_flops.to(TFLOPs/second).magnitude
h100_bw_tbs = Hardware.Cloud.H100.memory.bandwidth.to(TB/second).magnitude
h100_tflops_fp16_str = fmt(h100_tflops_fp16, precision=0, commas=False)
h100_bw_tbs_str = fmt_qty(h100_bw_tbs * TB / second, TB / second, precision=2, commas=False)
@@ -2841,10 +2841,10 @@ fig, ax, COLORS, plt = viz.setup_plot(figsize=(10, 4.5))
years = [2017, 2020, 2022, 2024]
chips = ['V100', 'A100', 'H100', 'B200']
ridges = [
Hardware.Cloud.V100.compute.peak_flops.m_as(TFLOPs/second) / Hardware.Cloud.V100.memory.bandwidth.m_as(TB/second),
Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second) / Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second),
Hardware.Cloud.H100.compute.peak_flops.m_as(TFLOPs/second) / Hardware.Cloud.H100.memory.bandwidth.m_as(TB/second),
Hardware.Cloud.B200.compute.peak_flops.m_as(TFLOPs/second) / Hardware.Cloud.B200.memory.bandwidth.m_as(TB/second),
Hardware.Cloud.V100.compute.peak_flops.to(TFLOPs/second).magnitude / Hardware.Cloud.V100.memory.bandwidth.to(TB/second).magnitude,
Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude / Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude,
Hardware.Cloud.H100.compute.peak_flops.to(TFLOPs/second).magnitude / Hardware.Cloud.H100.memory.bandwidth.to(TB/second).magnitude,
Hardware.Cloud.B200.compute.peak_flops.to(TFLOPs/second).magnitude / Hardware.Cloud.B200.memory.bandwidth.to(TB/second).magnitude,
]
# =============================================================================
@@ -3008,21 +3008,21 @@ class TensorLifecycleCalc:
# Energy value sourced from the mlsysim registry (Hardware.Tech.Memory).
ENERGY_DRAM_PJ_PER_BYTE = Hardware.Tech.Memory.DRAM.energy_per_byte
kws_samples_value = 16_000
kws_bytes_fp16_value = BYTES_FP16.m_as('B')
kws_bytes_fp16_value = BYTES_FP16.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
kws_tensor_kb_value = (kws_samples_value * BYTES_FP16).m_as(KB)
dram_energy_pj_bit_value = ENERGY_DRAM_PJ_PER_BYTE.m_as('pJ/B') / 8
a100_fp16_tflops_value = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
kws_tensor_kb_value = (kws_samples_value * BYTES_FP16).to(KB).magnitude
dram_energy_pj_bit_value = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude / 8
a100_fp16_tflops_value = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
kws_tensor_str = fmt_qty(kws_tensor_kb_value * KB, KB, precision=0, commas=False)
kws_samples_str = fmt(kws_samples_value, precision=0, commas=True)
kws_bytes_str = fmt(kws_bytes_fp16_value, precision=0, commas=False)
dram_energy_pj_bit_str = fmt(dram_energy_pj_bit_value, precision=0, commas=False)
latency_hbm_str = fmt(Hardware.Tech.Memory.DRAM.latency.m_as('ns'), precision=0, commas=False)
latency_l2_str = fmt(Hardware.Tech.Memory.L2.latency.m_as('ns'), precision=0, commas=False)
latency_l1_str = fmt(Hardware.Tech.Memory.L1.latency.m_as('ns'), precision=0, commas=False)
latency_hbm_str = fmt(Hardware.Tech.Memory.DRAM.latency.to(NS).magnitude, precision=0, commas=False)
latency_l2_str = fmt(Hardware.Tech.Memory.L2.latency.to(NS).magnitude, precision=0, commas=False)
latency_l1_str = fmt(Hardware.Tech.Memory.L1.latency.to(NS).magnitude, precision=0, commas=False)
a100_tflops_fp16_str = fmt_qty(a100_fp16_tflops_value * TFLOP / second, TFLOP / second, precision=0, commas=False)
# Plain-string literal → MarkdownStr per recipe Rule 1 (plain string literal _str)
reg_energy_pj_bit_str = MarkdownStr("0.1")
@@ -3486,20 +3486,20 @@ class InterconnectHierarchy:
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
# Device
hbm_bw = Hardware.Cloud.A100.memory.bandwidth.m_as(GB/second)
hbm_bw = Hardware.Cloud.A100.memory.bandwidth.to(GB/second).magnitude
# Chip-to-Chip
nvlink_a100 = Hardware.Cloud.A100.nvlink.bandwidth.m_as(GB/second)
nvlink_h100 = Hardware.Cloud.H100.nvlink.bandwidth.m_as(GB/second)
nvlink_a100 = Hardware.Cloud.A100.nvlink.bandwidth.to(GB/second).magnitude
nvlink_h100 = Hardware.Cloud.H100.nvlink.bandwidth.to(GB/second).magnitude
# Host-to-Device
pcie_gen4 = Hardware.Cloud.A100.interconnect.bandwidth.m_as(GB/second)
pcie_gen4 = Hardware.Cloud.A100.interconnect.bandwidth.to(GB/second).magnitude
# Node-to-Node (Network)
ib_hdr_gbps = Systems.Fabrics.InfiniBand_HDR.bandwidth.m_as(Gbps)
ib_hdr_gbs = Systems.Fabrics.InfiniBand_HDR.bandwidth.m_as(GB/second) # ~25 GB/s
ib_hdr_gbps = Systems.Fabrics.InfiniBand_HDR.bandwidth.to(Gbps).magnitude
ib_hdr_gbs = Systems.Fabrics.InfiniBand_HDR.bandwidth.to(GB/second).magnitude # ~25 GB/s
ib_ndr_gbps = Systems.Fabrics.InfiniBand_NDR.bandwidth.m_as(Gbps)
ib_ndr_gbps = Systems.Fabrics.InfiniBand_NDR.bandwidth.to(Gbps).magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
# The "bandwidth taper" must hold: HBM > NVLink > PCIe > Network
@@ -3514,10 +3514,10 @@ class InterconnectHierarchy:
# Bandwidth-taper equation — one consistent chip (H100-class) so HBM, NVLink,
# PCIe (Gen5), and the NDR network link cannot mix incompatible generations.
taper_hbm_gbs = Hardware.Cloud.H100.memory.bandwidth.m_as(GB/second)
taper_hbm_gbs = Hardware.Cloud.H100.memory.bandwidth.to(GB/second).magnitude
taper_nvlink_gbs = nvlink_h100
taper_pcie_gbs = Hardware.Cloud.H100.interconnect.bandwidth.m_as(GB/second)
taper_net_gbs = Systems.Fabrics.InfiniBand_NDR.bandwidth.m_as(GB/second)
taper_pcie_gbs = Hardware.Cloud.H100.interconnect.bandwidth.to(GB/second).magnitude
taper_net_gbs = Systems.Fabrics.InfiniBand_NDR.bandwidth.to(GB/second).magnitude
check(taper_hbm_gbs > taper_nvlink_gbs > taper_pcie_gbs > taper_net_gbs,
"bandwidth taper HBM > NVLink > PCIe > Network must hold")
taper_eq_math = MarkdownStr(
@@ -3644,7 +3644,7 @@ The key metric that determines which ceiling a workload hits is *arithmetic inte
# │ Goal: Demonstrate why newer accelerators are harder to saturate each generation.
# │ Show: Ridge points ~139 FLOP/byte (V100), ~153 (A100), ~296 (H100); rising gap.
# │ How: ridge_point() = peak_flops/memory_bw via Hardware Digital Twin; both in
# │ SI-compatible pint units so .m_as('flop/byte') extracts dimensionless ratio.
# │ SI-compatible pint units so .to(flop/byte).magnitude extracts dimensionless ratio.
# │
# │ Imports: mlsysim.core.constants (V100/A100/H100 peak_flops, memory_bw,
# │ flop, byte, GB, TB, TFLOPs, second)
@@ -3685,23 +3685,23 @@ class RooflineGap:
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
# Step 1: Ridge Points (FLOP/byte) directly from Twins
v100_ridge = h_v100.ridge_point().m_as('flop/byte')
a100_ridge = h_a100.ridge_point().m_as('flop/byte')
h100_ridge = h_h100.ridge_point().m_as('flop/byte')
v100_ridge = h_v100.ridge_point().to(flop/byte).magnitude
a100_ridge = h_a100.ridge_point().to(flop/byte).magnitude
h100_ridge = h_h100.ridge_point().to(flop/byte).magnitude
# Step 2: FP32 Ridge for A100
a100_ridge_fp32 = (h_a100.compute.precision_flops['fp32'] / h_a100.memory.bandwidth).m_as('flop/byte')
a100_ridge_fp32 = (h_a100.compute.precision_flops['fp32'] / h_a100.memory.bandwidth).to(flop/byte).magnitude
# Step 3: Display specs used by the nearby ridge-point prose
v100_tflops_value = h_v100.compute.peak_flops.m_as(TFLOPs/second)
v100_bw_tbs_value = h_v100.memory.bandwidth.m_as(TB/second)
a100_tflops_fp16_value = h_a100.compute.peak_flops.m_as(TFLOPs/second)
a100_bw_tbs_value = h_a100.memory.bandwidth.m_as(TB/second)
h100_tflops_fp16_value = h_h100.compute.peak_flops.m_as(TFLOPs/second)
h100_bw_tbs_value = h_h100.memory.bandwidth.m_as(TB/second)
v100_tflops_value = h_v100.compute.peak_flops.to(TFLOPs/second).magnitude
v100_bw_tbs_value = h_v100.memory.bandwidth.to(TB/second).magnitude
a100_tflops_fp16_value = h_a100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_bw_tbs_value = h_a100.memory.bandwidth.to(TB/second).magnitude
h100_tflops_fp16_value = h_h100.compute.peak_flops.to(TFLOPs/second).magnitude
h100_bw_tbs_value = h_h100.memory.bandwidth.to(TB/second).magnitude
# Step 4: Comparisons
bw_growth = h_h100.memory.bandwidth.m_as(GB/second) / h_a100.memory.bandwidth.m_as(GB/second)
bw_growth = h_h100.memory.bandwidth.to(GB/second).magnitude / h_a100.memory.bandwidth.to(GB/second).magnitude
flops_growth = h_h100.compute.peak_flops/h_a100.compute.peak_flops
relu_gap = h100_ridge/relu_ai
dense_matmul_flops = 2 * dense_matmul_dim ** 3
@@ -3838,28 +3838,28 @@ class TransformerLayerCalc:
t_batch_value = 32
t_seq_value = 512
t_heads_value = 12
t_fp_bytes_value = BYTES_FP16.m_as(byte)
t_fp_bytes_value = BYTES_FP16.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
# QKV Projection
qkv_flops_value = 2 * 3 * t_batch_value * t_seq_value * t_hidden_value * t_hidden_value
qkv_flops_b_value = (qkv_flops_value * flop).m_as(GFLOPs)
qkv_flops_b_value = (qkv_flops_value * flop).to(GFLOPs).magnitude
qkv_input_value = t_batch_value * t_seq_value * t_hidden_value
qkv_weights_value = 3 * t_hidden_value * t_hidden_value
qkv_output_value = t_batch_value * t_seq_value * t_hidden_value * 3
qkv_bytes_value = (qkv_input_value + qkv_weights_value + qkv_output_value) * t_fp_bytes_value
qkv_mb_value = (qkv_bytes_value * byte).m_as(MB)
qkv_mb_value = (qkv_bytes_value * byte).to(MB).magnitude
qkv_ai_value = qkv_flops_value/qkv_bytes_value
# Softmax
softmax_flops_value = t_batch_value * t_heads_value * t_seq_value * t_seq_value * 3
softmax_flops_m_value = (softmax_flops_value * flop).m_as(MFLOPs)
softmax_flops_m_value = (softmax_flops_value * flop).to(MFLOPs).magnitude
softmax_bytes_value = t_batch_value * t_heads_value * t_seq_value * t_seq_value * 2 * t_fp_bytes_value
softmax_mb_value = (softmax_bytes_value * byte).m_as(MB)
softmax_mb_value = (softmax_bytes_value * byte).to(MB).magnitude
softmax_ai_value = softmax_flops_value/softmax_bytes_value
a100_ridge_value = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
a100_ridge_value = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
t_hidden_str = fmt(t_hidden_value, precision=0, commas=False)
@@ -3944,23 +3944,23 @@ class Conv2dAnalysisCalc:
conv_w = 56
conv_cout = 256
conv_k = 3
conv_fp_bytes = BYTES_FP16.m_as('B')
conv_fp_bytes = BYTES_FP16.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
conv_out_elements = conv_batch * conv_cout * conv_h * conv_w
conv_out_m = conv_out_elements/MILLION
conv_flops_per_out = conv_cin * conv_k * conv_k * 2
conv_total_gflops = (conv_out_elements * conv_flops_per_out * flop).m_as(GFLOPs)
conv_total_gflops = (conv_out_elements * conv_flops_per_out * flop).to(GFLOPs).magnitude
conv_input_mb = (conv_batch * conv_cin * conv_h * conv_w * conv_fp_bytes * byte).m_as(MB)
conv_weights_mb = (conv_cout * conv_cin * conv_k * conv_k * conv_fp_bytes * byte).m_as(MB)
conv_output_mb = (conv_batch * conv_cout * conv_h * conv_w * conv_fp_bytes * byte).m_as(MB)
conv_input_mb = (conv_batch * conv_cin * conv_h * conv_w * conv_fp_bytes * byte).to(MB).magnitude
conv_weights_mb = (conv_cout * conv_cin * conv_k * conv_k * conv_fp_bytes * byte).to(MB).magnitude
conv_output_mb = (conv_batch * conv_cout * conv_h * conv_w * conv_fp_bytes * byte).to(MB).magnitude
conv_total_mb = conv_input_mb + conv_weights_mb + conv_output_mb
conv_ai = conv_total_gflops * THOUSAND / conv_total_mb
a100_ridge_value = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
a100_tflops_fp16_value = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_ridge_value = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
a100_tflops_fp16_value = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
conv_out_m_str = fmt(conv_out_m, precision=1, commas=False)
@@ -4040,21 +4040,21 @@ class DenseLayerAnalysisCalc:
dense_out = 2048
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
dense_total_mflops = (2 * dense_batch * dense_in * dense_out * flop).m_as(MFLOPs)
dense_total_mflops = (2 * dense_batch * dense_in * dense_out * flop).to(MFLOPs).magnitude
dense_input_kb = (dense_batch * dense_in * 2 * byte).m_as(KB)
dense_weights_mb = (dense_in * dense_out * 2 * byte).m_as(MB)
dense_output_kb = (dense_batch * dense_out * 2 * byte).m_as(KB)
dense_total_mb = (dense_input_kb * KB + dense_weights_mb * MB + dense_output_kb * KB).m_as(MB)
dense_input_kb = (dense_batch * dense_in * 2 * byte).to(KB).magnitude
dense_weights_mb = (dense_in * dense_out * 2 * byte).to(MB).magnitude
dense_output_kb = (dense_batch * dense_out * 2 * byte).to(KB).magnitude
dense_total_mb = (dense_input_kb * KB + dense_weights_mb * MB + dense_output_kb * KB).to(MB).magnitude
dense_ai = dense_total_mflops/dense_total_mb
a100_bw_gbs_value = Hardware.Cloud.A100.memory.bandwidth.m_as(GB/second)
a100_peak = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
dense_attainable_tflops = (a100_bw_gbs_value * dense_ai * GFLOPs).m_as(TFLOPs)
a100_bw_gbs_value = Hardware.Cloud.A100.memory.bandwidth.to(GB/second).magnitude
a100_peak = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
dense_attainable_tflops = (a100_bw_gbs_value * dense_ai * GFLOPs).to(TFLOPs).magnitude
dense_util_pct = dense_attainable_tflops/a100_peak * 100
a100_ridge_value = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
a100_ridge_value = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
dense_total_mflops_str = fmt(dense_total_mflops, precision=1, commas=False)
@@ -4140,16 +4140,16 @@ class LayernormAnalysisCalc:
ln_flops_per = 6
ln_total_mflops = ln_elements_m * ln_flops_per
ln_input_mb = (ln_elements * 2 * byte).m_as(MB)
ln_params_kb = (ln_hidden * 2 * 2 * byte).m_as(KB)
ln_output_mb = (ln_elements * 2 * byte).m_as(MB)
ln_total_mb = (ln_input_mb * MB + ln_output_mb * MB + ln_params_kb * KB).m_as(MB)
ln_input_mb = (ln_elements * 2 * byte).to(MB).magnitude
ln_params_kb = (ln_hidden * 2 * 2 * byte).to(KB).magnitude
ln_output_mb = (ln_elements * 2 * byte).to(MB).magnitude
ln_total_mb = (ln_input_mb * MB + ln_output_mb * MB + ln_params_kb * KB).to(MB).magnitude
ln_ai = ln_total_mflops/ln_total_mb
a100_bw_gbs_value = Hardware.Cloud.A100.memory.bandwidth.m_as(GB/second)
a100_ridge_value = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
ln_attainable_tflops = (a100_bw_gbs_value * ln_ai * GFLOPs).m_as(TFLOPs)
a100_bw_gbs_value = Hardware.Cloud.A100.memory.bandwidth.to(GB/second).magnitude
a100_ridge_value = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
ln_attainable_tflops = (a100_bw_gbs_value * ln_ai * GFLOPs).to(TFLOPs).magnitude
ln_ridge_gap = a100_ridge_value / ln_ai
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
@@ -4312,15 +4312,15 @@ class Gpt2ThroughputCalc:
gpt2_weight_gb = model_memory(Models.Language.GPT2.parameters, BYTES_FP16, GB)
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
gpt2_decode_flops = 2 * Models.Language.GPT2.parameters.m_as('param')
gpt2_decode_gflops = (gpt2_decode_flops * flop).m_as(GFLOPs)
gpt2_decode_flops = 2 * Models.Language.GPT2.parameters.to(param).magnitude
gpt2_decode_gflops = (gpt2_decode_flops * flop).to(GFLOPs).magnitude
gpt2_decode_ai = gpt2_decode_gflops/gpt2_weight_gb
a100_bw_tbs_val = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
a100_tflops_fp16_val = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_tflops_fp32_val = Hardware.Cloud.A100.compute.precision_flops["fp32"].m_as(TFLOPs/second)
a100_ridge_value = Hardware.Cloud.A100.ridge_point().m_as('flop/byte')
a100_ridge_fp32_value = (Hardware.Cloud.A100.compute.precision_flops['fp32'] / Hardware.Cloud.A100.memory.bandwidth).m_as('flop/byte')
a100_bw_tbs_val = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
a100_tflops_fp16_val = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_tflops_fp32_val = Hardware.Cloud.A100.compute.precision_flops["fp32"].to(TFLOPs/second).magnitude
a100_ridge_value = Hardware.Cloud.A100.ridge_point().to(flop/byte).magnitude
a100_ridge_fp32_value = (Hardware.Cloud.A100.compute.precision_flops['fp32'] / Hardware.Cloud.A100.memory.bandwidth).to(flop/byte).magnitude
gpt2_max_tflops = gpt2_decode_ai * a100_bw_tbs_val
gpt2_utilization = gpt2_max_tflops/a100_tflops_fp16_val * 100
@@ -4831,11 +4831,11 @@ class MemoryFootprintCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
tensor_dim = 1024
bytes_fp32 = BYTES_FP32.m_as('B')
bytes_fp32 = BYTES_FP32.to(byte).magnitude
n_intermediates = 4 # X, X', X'', Y tensors stored
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
tensor_mb = (tensor_dim * tensor_dim * bytes_fp32 * byte).m_as(MB)
tensor_mb = (tensor_dim * tensor_dim * bytes_fp32 * byte).to(MB).magnitude
total_mb = n_intermediates * tensor_mb
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -4920,10 +4920,10 @@ class FusionBenefitsCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
tensor_dim = 1024
bytes_per_float = BYTES_FP32.m_as('B')
bytes_per_float = BYTES_FP32.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
tensor_mb = (tensor_dim * tensor_dim * bytes_per_float * byte).m_as(MB)
tensor_mb = (tensor_dim * tensor_dim * bytes_per_float * byte).to(MB).magnitude
total_mb = tensor_mb * 4
naive_mb = total_mb # ~16.8 MB with all four intermediates stored
@@ -5489,7 +5489,7 @@ class RuntimeProductionTdp:
# (constant lookup, no derivation)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
a100_tdp_str = fmt(Hardware.Cloud.A100.tdp.m_as(watt), precision=0, commas=False)
a100_tdp_str = fmt(Hardware.Cloud.A100.tdp.to(watt).magnitude, precision=0, commas=False)
```
::: {#psp-hw-acceleration-when-production-differs-from-development .callout-perspective title="When production differs from development"}
@@ -5568,7 +5568,7 @@ class Gpt3SingleH100Calc:
peak_flops_per_device=Hardware.Cloud.H100.compute.precision_flops["fp8"],
efficiency_eta=1.0
)
peak_seconds = peak_seconds_qty.m_as(second)
peak_seconds = peak_seconds_qty.to(second).magnitude
peak_years = peak_seconds / SEC_PER_YEAR
realistic_util_low = 0.40
@@ -5580,7 +5580,7 @@ class Gpt3SingleH100Calc:
check(4.5 < peak_years < 5.5, f"Single-H100 FP8 peak time should be ~5 years, got {peak_years:.2f}")
# OUTPUT
h100_pflops_fp8_str = fmt_int(Hardware.Cloud.H100.compute.precision_flops["fp8"].m_as(PFLOPs/second), commas=False)
h100_pflops_fp8_str = fmt_int(Hardware.Cloud.H100.compute.precision_flops["fp8"].to(PFLOPs/second).magnitude, commas=False)
peak_years_str = fmt_int(round(peak_years), commas=False)
realistic_years_low_str = fmt(realistic_years_low, precision=1, commas=False)
realistic_years_high_str = fmt(realistic_years_high, precision=1, commas=False)
@@ -5717,8 +5717,8 @@ class FpWorkloadFitRecap:
"""A100 roofline recap for the hardware-specialization fallacy."""
# LOAD
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
softmax_ai_low = 2
softmax_ai_high = 5
cpu_ridge_low = 10
@@ -5783,8 +5783,8 @@ class FpMemoryEnergyCalc:
# Energy values sourced from the mlsysim registry (Hardware.Tech.Memory).
ENERGY_DRAM_ACCESS_PJ = Hardware.Tech.Memory.DRAM.energy_per_access
ENERGY_SRAM_L1_PJ = Hardware.Tech.Memory.L1.energy_per_access
dram_pj = ENERGY_DRAM_ACCESS_PJ.m_as('pJ')
sram_pj = ENERGY_SRAM_L1_PJ.m_as('pJ')
dram_pj = ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude
sram_pj = ENERGY_SRAM_L1_PJ.to(pJ).magnitude
layernorm_ai = 1.5
peak_tflops = 300 # hypothetical accelerator
@@ -5838,7 +5838,7 @@ class FpMultigpuScalingCalc:
"""NVLink gradient-sync overhead quantifying sublinear multi-GPU scaling."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
nvlink_bw_gbs = Hardware.Cloud.A100.nvlink.bandwidth.m_as(GB/second)
nvlink_bw_gbs = Hardware.Cloud.A100.nvlink.bandwidth.to(GB/second).magnitude
gradient_size_gb = 1.0
step_time_ms = 50
gpu_count = 8
@@ -5870,7 +5870,7 @@ class FpPeakVsSustainedRecap:
"""A100 peak and sustained ranges for the peak-FLOP/s fallacy."""
# LOAD
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
transformer_tflops_low = 120
transformer_tflops_high = 180
recsys_tflops_low = 10
@@ -5955,11 +5955,11 @@ class FpSmallBatchCalc:
ai_b256 = flops_b256 / bytes_b256
# T4 ridge point
t4_flops = Hardware.Cloud.T4.compute.peak_flops.m_as(TFLOPs/second)
t4_bw = Hardware.Cloud.T4.memory.bandwidth.m_as(GB/second) # for "GB/s" prose display
a100_bw = Hardware.Cloud.A100.memory.bandwidth.m_as(GB/second)
t4_bw_tbs = Hardware.Cloud.T4.memory.bandwidth.m_as(TB/second) # for dimensional roofline math
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
t4_flops = Hardware.Cloud.T4.compute.peak_flops.to(TFLOPs/second).magnitude
t4_bw = Hardware.Cloud.T4.memory.bandwidth.to(GB/second).magnitude # for "GB/s" prose display
a100_bw = Hardware.Cloud.A100.memory.bandwidth.to(GB/second).magnitude
t4_bw_tbs = Hardware.Cloud.T4.memory.bandwidth.to(TB/second).magnitude # for dimensional roofline math
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
t4_ridge = t4_flops / t4_bw_tbs # TFLOP/s / (TB/s) = FLOP/byte
a100_roof_b1 = a100_bw_tbs * ai_b1 # (TB/s) × (FLOP/byte) = TFLOP/s
t4_roof_b1 = t4_bw_tbs * ai_b1 # same
@@ -6203,8 +6203,8 @@ class SummaryRooflineRecap:
"""A100 ridge-point recap for the final takeaways."""
# LOAD
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.m_as(TB/second)
a100_peak_tflops = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_bw_tbs = Hardware.Cloud.A100.memory.bandwidth.to(TB/second).magnitude
# EXECUTE
a100_ridge = a100_peak_tflops / a100_bw_tbs
book/quarto/contents/vol1/ml_ops/ml_ops.qmd
+15 -12
View File
@@ -2110,7 +2110,7 @@ class KVCacheFootprint:
batch_size=1,
bytes_per_elem=bytes_per_element
)
kv_cache_gb = kv_cache_one.m_as(GB)
kv_cache_gb = kv_cache_one.to(GB).magnitude
concurrent_8_gb = 8 * kv_cache_gb
concurrent_8_q = 8 * kv_cache_one
@@ -2124,7 +2124,7 @@ class KVCacheFootprint:
kv_heads_str = fmt(kv_heads, precision=0, commas=False)
seq_len_str = fmt(seq_len, precision=0, commas=True)
head_dim_str = fmt(head_dim, precision=0, commas=False)
bytes_per_element_str = fmt(bytes_per_element.m_as(byte), precision=0, commas=False)
bytes_per_element_str = fmt(bytes_per_element.to(byte).magnitude, precision=0, commas=False)
kv_cache_gb_str = fmt_qty(kv_cache_one, GB, precision=1, commas=False)
concurrent_8_gb_str = fmt_qty(concurrent_8_q, GB, precision=1, commas=False)
```
@@ -2440,13 +2440,13 @@ class ObservabilitySampling:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
high_bytes_per_second = requests_per_second * telemetry_per_request / high_frequency_seconds
low_bytes_per_second = requests_per_second * telemetry_per_request / low_frequency_seconds
sampling_ratio = (high_bytes_per_second / low_bytes_per_second).m_as("")
sampling_ratio = (high_bytes_per_second / low_bytes_per_second).to("").magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(round(high_bytes_per_second.m_as(GB), 1) == 1.0,
f"Expected ~1 GB/s, got {high_bytes_per_second.m_as(GB):.2f} GB/s.")
check(round(low_bytes_per_second.m_as(MB), 1) == 16.7,
f"Expected ~16.7 MB/s, got {low_bytes_per_second.m_as(MB):.2f} MB/s.")
check(round(high_bytes_per_second.to(GB).magnitude, 1) == 1.0,
f"Expected ~1 GB/s, got {high_bytes_per_second.to(GB).magnitude:.2f} GB/s.")
check(round(low_bytes_per_second.to(MB).magnitude, 1) == 16.7,
f"Expected ~16.7 MB/s, got {low_bytes_per_second.to(MB).magnitude:.2f} MB/s.")
check(round(sampling_ratio) == 60,
f"Expected 60x sampling ratio, got {sampling_ratio:.1f}x.")
@@ -2741,7 +2741,7 @@ class MonitoringBudget:
datapoints_mo_m = datapoints_mo/MILLION
ingestion_cost = datapoints_mo_m * ingestion_cost_per_m
storage_q = datapoints_mo * bytes_per_point * byte
storage_gb = storage_q.m_as(GB)
storage_gb = storage_q.to(GB).magnitude
storage_cost = storage_gb * storage_cost_per_gb
queries_mo = n_dashboards * n_users * (queries_per_hr * work_hours * work_days)
query_cost = queries_mo * query_cost_per
@@ -3200,7 +3200,7 @@ Resource justification requires translating technical requirements into business
# │ Imports: mlsysim.book (fmt)
# │ Exports: current_rate_pct_str, target_rate_pct_str, annual_loss_prevented_str, etc.
# └─────────────────────────────────────────────────────────────────────────────
from mlsysim.fmt import fmt, check, MarkdownStr
from mlsysim.fmt import check, fmt_percent, fmt_percent_range, fmt_pp
# ┌── LEGO ───────────────────────────────────────────────
class FraudDetectionImprovement:
@@ -3611,9 +3611,12 @@ class OuraValidationGap:
scorer_low_pct_str = fmt_percent(scorer_agreement_low, precision=0, commas=False, style='prose')
scorer_high_pct_str = fmt_percent(scorer_agreement_high, precision=0, commas=False, style='prose')
gain_points_str = fmt_pp(accuracy_gain * 100, precision=0, commas=False, style="symbol")
gap_closed_range_str = MarkdownStr(
f"{fmt(gap_closed_high_ceiling * 100, precision=1, commas=False)}-"
f"{fmt(gap_closed_low_ceiling * 100, precision=0, commas=False)} percent"
gap_closed_range_str = fmt_percent_range(
gap_closed_high_ceiling,
gap_closed_low_ceiling,
precision=(1, 0),
commas=False,
style="prose",
)
```
book/quarto/contents/vol1/model_compression/model_compression.qmd
+58 -58
View File
@@ -61,17 +61,17 @@ from mlsysim import Models, Platforms
from mlsysim.fmt import fmt, fmt_int, fmt_qty, fmt_qty_int, check
class CompressionDeploymentScale:
llm_7b_params = Models.Language.Llama2_7B.parameters.m_as(Bparam)
bytes_fp16 = BYTES_FP16.m_as(byte)
llm_7b_params = Models.Language.Llama2_7B.parameters.to(Bparam).magnitude
bytes_fp16 = BYTES_FP16.to(byte).magnitude
llm_7b_mem_fp16_gb = llm_7b_params * bytes_fp16
llm_7b_mem_fp16 = llm_7b_mem_fp16_gb * GB
llm_175b_params = Models.Language.GPT3.parameters.m_as(Bparam)
llm_175b_params = Models.Language.GPT3.parameters.to(Bparam).magnitude
llm_175b_mem_fp16_gb = llm_175b_params * bytes_fp16
llm_175b_mem_fp16 = llm_175b_mem_fp16_gb * GB
smartphone_ram_gb = Platforms.Mobile.ram.m_as(GB)
smartphone_ram_gb = Platforms.Mobile.ram.to(GB).magnitude
smartphone_ram = round(smartphone_ram_gb) * GB
mcu_ram_kb = Platforms.Tiny.ram.m_as(KB)
mcu_ram_kb = Platforms.Tiny.ram.to(KB).magnitude
mcu_ram = mcu_ram_kb * KB
check(llm_7b_mem_fp16_gb > smartphone_ram_gb,
@@ -112,9 +112,9 @@ from mlsysim import Models
from mlsysim.fmt import fmt, fmt_qty, check, fmt_count
class CompressionIronLawQuant:
bytes_fp16 = BYTES_FP16.m_as(byte)
bytes_int8 = BYTES_INT8.m_as(byte)
llm_175b_params = Models.Language.GPT3.parameters.m_as(Bparam)
bytes_fp16 = BYTES_FP16.to(byte).magnitude
bytes_int8 = BYTES_INT8.to(byte).magnitude
llm_175b_params = Models.Language.GPT3.parameters.to(Bparam).magnitude
llm_175b_mem_fp16_gb = llm_175b_params * bytes_fp16
llm_175b_mem_int8_gb = llm_175b_params * bytes_int8
llm_175b_mem_fp16 = llm_175b_mem_fp16_gb * GB
@@ -229,17 +229,17 @@ class QuantizationEnergyPhysics:
ENERGY_ADD_INT8_PJ = Hardware.Tech.Op.AddInt8.energy
int8_energy_reduction = 20
bytes_fp32 = BYTES_FP32.m_as(byte)
bytes_int8 = BYTES_INT8.m_as(byte)
bytes_fp32 = BYTES_FP32.to(byte).magnitude
bytes_int8 = BYTES_INT8.to(byte).magnitude
energy_dram = ENERGY_DRAM_ACCESS_PJ.m_as(ureg.picojoule)
energy_dram_per_byte = ENERGY_DRAM_PJ_PER_BYTE.m_as(ureg.picojoule/ureg.byte)
energy_dram = ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude
energy_dram_per_byte = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude
energy_dram_read_bytes = energy_dram / energy_dram_per_byte
energy_dram_read_bits = energy_dram_read_bytes * 8
energy_flop_fp32 = ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule/ureg.count)
energy_op_int8 = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count)
energy_add_fp32 = ENERGY_ADD_FP32_PJ.m_as(ureg.picojoule)
energy_add_int8 = ENERGY_ADD_INT8_PJ.m_as(ureg.picojoule)
energy_flop_fp32 = ENERGY_FLOP_FP32_PJ.to(pJ).magnitude
energy_op_int8 = ENERGY_OP_INT8_PJ.to(pJ).magnitude
energy_add_fp32 = ENERGY_ADD_FP32_PJ.to(pJ).magnitude
energy_add_int8 = ENERGY_ADD_INT8_PJ.to(pJ).magnitude
energy_float_add_ratio = energy_add_fp32 / energy_add_int8
energy_dram_read_ratio = energy_dram / energy_add_int8
@@ -386,13 +386,13 @@ class QuantizationSpeedup:
"""
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
params_b = Models.Language.Llama2_7B.parameters.m_as(Bparam)
bytes_fp16 = BYTES_FP16.m_as(byte)
bytes_int4 = BYTES_INT4.m_as(byte)
params_b = Models.Language.Llama2_7B.parameters.to(Bparam).magnitude
bytes_fp16 = BYTES_FP16.to(byte).magnitude
bytes_int4 = BYTES_INT4.to(byte).magnitude
device_ram_gb = 16
device_ram = device_ram_gb * GB
mem_bw_gbs = Hardware.Tech.Storage.SystemMemory.bandwidth.m_as(GB / second)
mem_bw_gbs = Hardware.Tech.Storage.SystemMemory.bandwidth.to(GB / second).magnitude
mem_bw = Hardware.Tech.Storage.SystemMemory.bandwidth
kv_cache_gb = 1.0
kv_cache = kv_cache_gb * GB
@@ -542,7 +542,7 @@ from mlsysim.fmt import fmt, fmt_int, MarkdownStr, fmt_qty, fmt_qty_int
def _get_ratio(model_mem, device_mem):
"""Return 'ok' if model fits, else 'no (Nx)' with how many times it overflows."""
ratio = model_mem.m_as(byte) / device_mem.m_as(byte)
ratio = model_mem.to(byte).magnitude / device_mem.to(byte).magnitude
if ratio < 1:
return "ok"
return f"no ({ratio:.0f}x)"
@@ -572,7 +572,7 @@ class ModelDeviceComparison:
resnet_tiny = _get_ratio(resnet_mem, tiny_mem)
mobilenet_tiny = _get_ratio(mobilenet_mem, tiny_mem)
mobilenet_int8_tiny = _get_ratio(mobilenet_int8_mem, tiny_mem)
mobilenet_int8_tiny_ratio = mobilenet_int8_mem.m_as(byte) / tiny_mem.m_as(byte)
mobilenet_int8_tiny_ratio = mobilenet_int8_mem.to(byte).magnitude / tiny_mem.to(byte).magnitude
# ┌── 3. GUARD (Invariants) ──────────────────────────────────────────
# DS-CNN always fits TinyML—sanity check
@@ -2372,12 +2372,12 @@ class LowRankFactorization:
# Step 1: Full Matrix: N * N
full_params = mat_dim * mat_dim
full = full_params * bytes_per_param * byte
full_mb = full.m_as(MB)
full_mb = full.to(MB).magnitude
# Step 2: Factored: 2 * N * K
factored_params = 2 * mat_dim * rank_k
factored = factored_params * bytes_per_param * byte
factored_mb = factored.m_as(MB)
factored_mb = factored.to(MB).magnitude
data_reduction = full_mb/factored_mb
@@ -2996,9 +2996,9 @@ class QuantizationMemoryAnchor:
"""Local 7-billion-parameter FP16 memory anchor for quantization motivation."""
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
llm_7b_params_b = Models.Language.Llama2_7B.parameters.m_as(Bparam)
llm_7b_params_b = Models.Language.Llama2_7B.parameters.to(Bparam).magnitude
bytes_fp16 = 2
smartphone_ram_gb = Platforms.Mobile.ram.m_as(GB)
smartphone_ram_gb = Platforms.Mobile.ram.to(GB).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
llm_7b_mem_fp16_gb = llm_7b_params_b * bytes_fp16
@@ -3070,10 +3070,10 @@ class EnergyCosts:
ENERGY_ADD_INT32_PJ = Hardware.Tech.Op.AddInt32.energy
ENERGY_ADD_INT8_PJ = Hardware.Tech.Op.AddInt8.energy
energy_add_fp32_pj = ENERGY_ADD_FP32_PJ.m_as(ureg.picojoule)
energy_add_fp16_pj = ENERGY_ADD_FP16_PJ.m_as(ureg.picojoule)
energy_add_int32_pj = ENERGY_ADD_INT32_PJ.m_as(ureg.picojoule)
energy_add_int8_pj = ENERGY_ADD_INT8_PJ.m_as(ureg.picojoule)
energy_add_fp32_pj = ENERGY_ADD_FP32_PJ.to(pJ).magnitude
energy_add_fp16_pj = ENERGY_ADD_FP16_PJ.to(pJ).magnitude
energy_add_int32_pj = ENERGY_ADD_INT32_PJ.to(pJ).magnitude
energy_add_int8_pj = ENERGY_ADD_INT8_PJ.to(pJ).magnitude
# ┌── 2. EXECUTE—(values are direct from constants) ─────────────────
@@ -3208,12 +3208,12 @@ class EnergyDividend:
ENERGY_DRAM_PJ_PER_BYTE = Hardware.Tech.Memory.DRAM.energy_per_byte
ENERGY_OP_INT8_PJ = Hardware.Tech.Op.OpInt8.energy
e_add_fp32 = ENERGY_ADD_FP32_PJ.m_as('pJ')
e_add_int8 = ENERGY_ADD_INT8_PJ.m_as('pJ')
e_dram = ENERGY_DRAM_ACCESS_PJ.m_as(ureg.picojoule)
e_dram_per_byte = ENERGY_DRAM_PJ_PER_BYTE.m_as(ureg.picojoule/ureg.byte)
e_add_fp32 = ENERGY_ADD_FP32_PJ.to(pJ).magnitude
e_add_int8 = ENERGY_ADD_INT8_PJ.to(pJ).magnitude
e_dram = ENERGY_DRAM_ACCESS_PJ.to(pJ).magnitude
e_dram_per_byte = ENERGY_DRAM_PJ_PER_BYTE.to(pJ / byte).magnitude
dram_read_bits = (e_dram / e_dram_per_byte) * 8
e_int8_mac = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count) + e_add_int8
e_int8_mac = ENERGY_OP_INT8_PJ.to(pJ).magnitude + e_add_int8
dividend = e_add_fp32 / e_add_int8
dram_read_vs_int8_mac = e_dram / e_int8_mac
@@ -3382,7 +3382,7 @@ To make these gains concrete, consider the quantization savings when deploying a
# │ fp16_size_gb_str, int4_size_gb_str, compression_ratio_str
# └─────────────────────────────────────────────────────────────────────────────
from mlsysim import Models
from mlsysim.fmt import fmt_int, fmt, fmt_qty, fmt_qty_int, check, MarkdownStr
from mlsysim.fmt import fmt_int, fmt, fmt_qty, fmt_qty_int, check
# ┌── LEGO ───────────────────────────────────────────────
class QuantizationSavings:
@@ -3392,11 +3392,11 @@ class QuantizationSavings:
"""
# ┌── 1. LOAD (Constants) ───────────────────────────────────────────────
params_b = round(Models.Language.Llama3_8B.parameters.m_as(Bparam))
params_b = round(Models.Language.Llama3_8B.parameters.to(Bparam).magnitude)
gpu_mem_fp16_gb = 24
gpu_mem_int4_gb = 8
bytes_fp16 = BYTES_FP16.m_as(byte)
bytes_int4 = BYTES_INT4.m_as(byte)
bytes_fp16 = BYTES_FP16.to(byte).magnitude
bytes_int4 = BYTES_INT4.to(byte).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
fp16_size_gb = params_b * bytes_fp16
@@ -3410,7 +3410,7 @@ class QuantizationSavings:
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(ratio == 4.0, f"FP16/INT4 ratio should be exactly 4.0, got {ratio}")
check(BYTES_FP32.m_as(ureg.byte) / BYTES_INT8.m_as(ureg.byte) == 4.0, "Narrative Violation: INT8 compression ratio is no longer exactly 4x! Got ratio.")
check(BYTES_FP32.to(byte).magnitude / BYTES_INT8.to(byte).magnitude == 4.0, "Narrative Violation: INT8 compression ratio is no longer exactly 4x! Got ratio.")
# ┌── 4. OUTPUT (Formatting) ──────────────────────────────────────────────
llm_params_b_str = fmt(params_b, precision=0, commas=False)
@@ -3419,8 +3419,8 @@ class QuantizationSavings:
fp16_size_gb_str = fmt_qty_int(fp16_size, GB, commas=False)
int4_size_gb_str = fmt_qty_int(int4_size, GB, commas=False)
compression_ratio_str = fmt_int(ratio, commas=False)
gpu_mem_fp16_gb_str = MarkdownStr(f"{fmt_qty(gpu_mem_fp16, GB, precision=0, commas=False)} GPU")
gpu_mem_int4_gb_str = MarkdownStr(f"{fmt_qty(gpu_mem_int4, GB, precision=0, commas=False)} GPU")
gpu_mem_fp16_gb_str = fmt_qty(gpu_mem_fp16, GB, precision=0, commas=False)
gpu_mem_int4_gb_str = fmt_qty(gpu_mem_int4, GB, precision=0, commas=False)
```
::: {#nbk-model-compression-quantization-savings .callout-notebook title="Quantization savings"}
@@ -3430,12 +3430,12 @@ class QuantizationSavings:
**FP16 (Half Precision)**
- **Size**: `{python} QuantizationSavings.llm_params_b_str` $\times 10^9 \times$ `{python} QuantizationSavings.fp16_bytes_str` bytes (16-bit) = `{python} QuantizationSavings.fp16_size_gb_str`
- **Hardware Req**: Requires `{python} QuantizationSavings.gpu_mem_fp16_gb_str` (for example, A10G, 3090, 4090).
- **Hardware Req**: Requires `{python} QuantizationSavings.gpu_mem_fp16_gb_str` GPU (for example, A10G, 3090, 4090).
**INT4 (4-bit Quantization)**
- **Size**: `{python} QuantizationSavings.llm_params_b_str` $\times 10^9 \times$ `{python} QuantizationSavings.int4_bytes_str` bytes (INT4) = `{python} QuantizationSavings.int4_size_gb_str`
- **Hardware Req**: Fits comfortably on `{python} QuantizationSavings.gpu_mem_int4_gb_str` (for example, T4, consumer laptops).
- **Hardware Req**: Fits comfortably on `{python} QuantizationSavings.gpu_mem_int4_gb_str` GPU (for example, T4, consumer laptops).
**Impact**: `{python} QuantizationSavings.compression_ratio_str`$\times$ compression allows deployment on commodity hardware instead of requiring a larger accelerator primarily for weight storage.
@@ -3653,8 +3653,8 @@ class A100Int8Speedup:
"""A100 Tensor Core throughput: FP16 vs INT8."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.m_as(TFLOPs/second)
a100_tops_int8 = Hardware.Cloud.A100.compute.precision_flops["int8"].m_as(TOPS)
a100_tflops_fp16 = Hardware.Cloud.A100.compute.peak_flops.to(TFLOPs/second).magnitude
a100_tops_int8 = Hardware.Cloud.A100.compute.precision_flops["int8"].to(TOPS).magnitude
a100_fp16 = Hardware.Cloud.A100.compute.peak_flops
a100_int8 = Hardware.Cloud.A100.compute.precision_flops["int8"]
bandwidth_bound_speedup = 2 # FP16 -> INT8 memory-traffic-dominated case
@@ -3792,7 +3792,7 @@ from mlsysim.core.constants import Bparam
from mlsysim.fmt import fmt
class QuantizationSavingsRecap:
params_b = round(Models.Language.Llama3_8B.parameters.m_as(Bparam))
params_b = round(Models.Language.Llama3_8B.parameters.to(Bparam).magnitude)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
llm_params_b_str = fmt(params_b, precision=0, commas=False)
```
@@ -3830,10 +3830,10 @@ class EnergyRatio:
ENERGY_ADD_FP32_PJ = Hardware.Tech.Op.AddFp32.energy
ENERGY_ADD_INT8_PJ = Hardware.Tech.Op.AddInt8.energy
energy_mul_fp32_pj = ENERGY_FLOP_FP32_PJ.m_as(ureg.picojoule/ureg.count)
energy_add_fp32_pj = ENERGY_ADD_FP32_PJ.m_as(ureg.picojoule)
energy_op_int8_pj = ENERGY_OP_INT8_PJ.m_as(ureg.picojoule/ureg.count)
energy_add_int8_pj = ENERGY_ADD_INT8_PJ.m_as(ureg.picojoule)
energy_mul_fp32_pj = ENERGY_FLOP_FP32_PJ.to(pJ).magnitude
energy_add_fp32_pj = ENERGY_ADD_FP32_PJ.to(pJ).magnitude
energy_op_int8_pj = ENERGY_OP_INT8_PJ.to(pJ).magnitude
energy_add_int8_pj = ENERGY_ADD_INT8_PJ.to(pJ).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
fp32_mac_pj = energy_mul_fp32_pj + energy_add_fp32_pj # ~4.6 pJ
@@ -5274,7 +5274,7 @@ class FusionCalc:
kernels_unfused = 159
kernels_fused = 53
latency_per_kernel_us = 10
v100_bw_gbs = Hardware.Cloud.V100.memory.bandwidth.m_as(GB/second)
v100_bw_gbs = Hardware.Cloud.V100.memory.bandwidth.to(GB/second).magnitude
feat_map_mb = conv_channels * conv_spatial * conv_spatial * bytes_per_element/MILLION
feat_map = feat_map_mb * MB
@@ -7390,16 +7390,16 @@ class ResNet50Int8Metrics:
p_int8 = Engine.solve(m, h, batch_size=1, precision="int8")
warnings.filters[:] = _prev_filters
fp32_latency_ms = p_fp32.latency.m_as("ms")
int8_latency_ms = p_int8.latency.m_as("ms")
fp32_latency_ms = p_fp32.latency.to(ms).magnitude
int8_latency_ms = p_int8.latency.to(ms).magnitude
fp32_size = p_fp32.memory_footprint
int8_size = p_int8.memory_footprint
fp32_size_mb = fp32_size.m_as("MB")
int8_size_mb = int8_size.m_as("MB")
fp32_size_mb = fp32_size.to(MB).magnitude
int8_size_mb = int8_size.to(MB).magnitude
fp32_energy = p_fp32.energy
int8_energy = p_int8.energy
fp32_energy_j = fp32_energy.m_as("J")
int8_energy_j = int8_energy.m_as("J")
fp32_energy_j = fp32_energy.to(joule).magnitude
int8_energy_j = int8_energy.to(joule).magnitude
top1_drop = fp32_top1 - int8_top1
latency_speedup = fp32_latency_ms / int8_latency_ms
book/quarto/contents/vol1/model_serving/model_serving.qmd
+48 -48
View File
@@ -761,7 +761,7 @@ class StaticBatchCalc:
dynamic_latency_budget_ms_value = 100 # real-time latency budget (ms)
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
batch_total_s_value = (n_photos_value * inference_ms_value * ms).m_as(second)
batch_total_s_value = (n_photos_value * inference_ms_value * ms).to(second).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
n_photos_str = fmt_count(n_photos_value, label="photo")
@@ -944,7 +944,7 @@ class ResNetServingSpectrum:
cloud_inf_b16_ms = 14.0
cloud_throughput = 1143
cloud_vram = 2 * GB
cloud_vram_gb = cloud_vram.m_as(GB)
cloud_vram_gb = cloud_vram.to(GB).magnitude
# Mobile (Smartphone) Performance
mobile_inf_npu_ms = 12.0
@@ -952,13 +952,13 @@ class ResNetServingSpectrum:
mobile_throughput = 80
mobile_energy_npu = 0.8 * ureg.millijoule
mobile_energy_cpu = 4.2 * ureg.millijoule
mobile_energy_npu_mj = mobile_energy_npu.m_as(ureg.millijoule)
mobile_energy_cpu_mj = mobile_energy_cpu.m_as(ureg.millijoule)
mobile_energy_npu_mj = mobile_energy_npu.to(mJ).magnitude
mobile_energy_cpu_mj = mobile_energy_cpu.to(mJ).magnitude
# TinyML (Cortex-M7) Performance
tiny_inf_ms = 120.0
tiny_energy = 12.0 * ureg.millijoule
tiny_energy_mj = tiny_energy.m_as(ureg.millijoule)
tiny_energy_mj = tiny_energy.to(mJ).magnitude
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
# Step 1: Calculate sizes using the Digital Twins
@@ -966,13 +966,13 @@ class ResNetServingSpectrum:
mobile_size = m_resnet.size_in_bytes(BYTES_INT8)
tiny_original = m_resnet.size_in_bytes(BYTES_INT8)
tiny_alt = m_mobilenet.size_in_bytes(BYTES_INT8)
cloud_size_mb = cloud_size.m_as('MB')
mobile_size_mb = mobile_size.m_as('MB')
tiny_original_mb = tiny_original.m_as('MB')
tiny_alt_mb = tiny_alt.m_as('MB')
cloud_size_mb = cloud_size.to(MB).magnitude
mobile_size_mb = mobile_size.to(MB).magnitude
tiny_original_mb = tiny_original.to(MB).magnitude
tiny_alt_mb = tiny_alt.to(MB).magnitude
# Step 2: TinyML feasibility check
tiny_limit_mb = s_tiny.ram.m_as('MB')
tiny_limit_mb = s_tiny.ram.to(MB).magnitude
tiny_feasibility = tiny_original_mb < tiny_limit_mb
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
@@ -2146,7 +2146,7 @@ class CapacityPlanningAnchor:
"""
qps_target = 1000
slo_ms = 50
concurrency_slots = int(qps_target * (slo_ms * ms).m_as(second))
concurrency_slots = int(qps_target * (slo_ms * ms).to(second).magnitude)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
qps_str = fmt_rate(qps_target, "QPS", commas=False)
@@ -2266,12 +2266,12 @@ class BatchingTax:
# ┌── 2. EXECUTE (The Compute) ─────────────────────────────────────────
# Step 1: Batch 1
w_form_b1 = ((1-1) / (2 * lambda_qps) * second).m_as(ms) # 0 ms
w_form_b1 = ((1-1) / (2 * lambda_qps) * second).to(ms).magnitude # 0 ms
lat_b1 = w_form_b1 + t_inf_b1
# Batch 32
# Step 2: Formation Delay ~ (B-1) / (2 * lambda)
w_form_b32 = ((32-1) / (2 * lambda_qps) * second).m_as(ms) # ~31 ms
w_form_b32 = ((32-1) / (2 * lambda_qps) * second).to(ms).magnitude # ~31 ms
lat_b32 = w_form_b32 + t_inf_b32
penalty_ratio = lat_b32 / lat_b1
@@ -2711,12 +2711,12 @@ class ModelSwapCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
model_size = 10 * GB # model size
model_size_gb_value = model_size.m_as(GB)
model_size_gb_value = model_size.to(GB).magnitude
pcie_bw = Hardware.Cloud.A100.interconnect.bandwidth # PCIe Gen4 x16 bandwidth
pcie_bw_gbs_value = pcie_bw.m_as(GB/second)
pcie_bw_gbs_value = pcie_bw.to(GB/second).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
model_swap_ms_value = (model_size_gb_value / pcie_bw_gbs_value * second).m_as(ms)
model_swap_ms_value = (model_size_gb_value / pcie_bw_gbs_value * second).to(ms).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
model_size_gb_str = fmt_qty(model_size, GB, precision=0, commas=False)
@@ -3037,7 +3037,7 @@ def total_latency_value(b, _T=10.0, _fixed=5.0, _per_image=0.6):
return _T + _fixed + _per_image * b
def throughput_value(b, _T=10.0, _fixed=5.0, _per_image=0.6):
return b / ((_T + _fixed + _per_image * b) * ms).m_as(second)
return b / ((_T + _fixed + _per_image * b) * ms).to(second).magnitude
class BatchingAnalysisCalc:
"""Quantifies batching efficiency: batch-32 achieves 14.6× throughput gain over batch-1."""
@@ -3260,7 +3260,7 @@ class SloViolationCalc:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
# service_time_value is a module-level function from batching-analysis-calc
mean_batch_value = qps_value * (T_slo_value * ms).m_as(second)
mean_batch_value = qps_value * (T_slo_value * ms).to(second).magnitude
mean_wait_value = T_slo_value / 2
mean_service_value = service_time_value(int(mean_batch_value))
mean_latency_value = mean_wait_value + mean_service_value
@@ -3407,7 +3407,7 @@ class PracticalConfigCalc:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
pc_batch_budget_ms_value = pc_slo_ms_value * pc_budget_pct_value
pc_max_window_ms_value = pc_batch_budget_ms_value
pc_window_seconds_value = (pc_config_window_ms_value * ms).m_as(second)
pc_window_seconds_value = (pc_config_window_ms_value * ms).to(second).magnitude
pc_expected_batch_value = pc_qps_value * pc_window_seconds_value
poisson_term = math.exp(-pc_expected_batch_value)
@@ -3424,7 +3424,7 @@ class PracticalConfigCalc:
pc_peak_service_ms_value = pc_base_service_ms_value + pc_per_image_ms_value * pc_config_batch_value
pc_predicted_throughput_value = (
pc_config_batch_value / (pc_peak_service_ms_value * ms).m_as(second) * pc_throughput_efficiency_value
pc_config_batch_value / (pc_peak_service_ms_value * ms).to(second).magnitude * pc_throughput_efficiency_value
)
check(pc_predicted_p99_ms_value < pc_slo_ms_value, "Predicted p99 should stay within the serving SLO.")
@@ -3563,13 +3563,13 @@ class TrafficAdaptiveBatchingCalc:
arrival_rates = (100, 500, 1_000, 5_000)
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
service_time_s = (service_time_ms * ms).m_as(second)
latency_budget_s = ((latency_slo_ms - service_time_ms) * ms).m_as(second)
service_time_s = (service_time_ms * ms).to(second).magnitude
latency_budget_s = ((latency_slo_ms - service_time_ms) * ms).to(second).magnitude
rows = []
for qps in arrival_rates:
window_s = min(latency_budget_s, math.sqrt(service_time_s / qps))
avg_batch = qps * window_s
approx_latency_ms = service_time_ms + (window_s / (1 * ms).m_as(second))
approx_latency_ms = service_time_ms + (window_s / (1 * ms).to(second).magnitude)
rows.append((qps, window_s, avg_batch, approx_latency_ms))
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
@@ -3584,22 +3584,22 @@ class TrafficAdaptiveBatchingCalc:
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
qps_100_str = fmt_rate(rows[0][0], "QPS")
window_100_ms_str = fmt_time(rows[0][1] / (1 * ms).m_as(second), 'millisecond', precision=1, commas=False)
window_100_ms_str = fmt_time(rows[0][1] / (1 * ms).to(second).magnitude, 'millisecond', precision=1, commas=False)
avg_batch_100_str = fmt(rows[0][2], precision=1, commas=False)
latency_100_ms_str = fmt_time(rows[0][3], 'millisecond', precision=1, commas=False)
qps_500_str = fmt_rate(rows[1][0], "QPS")
window_500_ms_str = fmt_time(rows[1][1] / (1 * ms).m_as(second), 'millisecond', precision=1, commas=False)
window_500_ms_str = fmt_time(rows[1][1] / (1 * ms).to(second).magnitude, 'millisecond', precision=1, commas=False)
avg_batch_500_str = fmt(rows[1][2], precision=1, commas=False)
latency_500_ms_str = fmt_time(rows[1][3], 'millisecond', precision=1, commas=False)
qps_1000_str = fmt_rate(rows[2][0], "QPS")
window_1000_ms_str = fmt_time(rows[2][1] / (1 * ms).m_as(second), 'millisecond', precision=0, commas=False)
window_1000_ms_str = fmt_time(rows[2][1] / (1 * ms).to(second).magnitude, 'millisecond', precision=0, commas=False)
avg_batch_1000_str = fmt(rows[2][2], precision=0, commas=False)
latency_1000_ms_str = fmt_time(rows[2][3], 'millisecond', precision=0, commas=False)
qps_5000_str = fmt_rate(rows[3][0], "QPS")
window_5000_ms_str = fmt_time(rows[3][1] / (1 * ms).m_as(second), 'millisecond', precision=1, commas=False)
window_5000_ms_str = fmt_time(rows[3][1] / (1 * ms).to(second).magnitude, 'millisecond', precision=1, commas=False)
avg_batch_5000_str = fmt(rows[3][2], precision=1, commas=False)
latency_5000_ms_str = fmt_time(rows[3][3], 'millisecond', precision=1, commas=False)
```
@@ -3714,7 +3714,7 @@ class MobileServingCalc:
"mobile pipeline sustained power should be ~60–70 mW",
)
inferences_per_battery_m = (
(battery_wh / m_total_mj).m_as(ureg.count) / MILLION
(battery_wh / m_total_mj).to(count).magnitude / MILLION
)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -3902,7 +3902,7 @@ class CarbonCostH100Tdp:
h_h100 = Hardware.Cloud.H100
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
h100_tdp_value = h_h100.tdp.m_as(watt)
h100_tdp_value = h_h100.tdp.to(watt).magnitude
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
h100_tdp_str = fmt(h100_tdp_value, precision=0, commas=False) # e.g. "700" W
@@ -3953,14 +3953,14 @@ class CarbonCostCalc:
cc_low_util_joules = cc_idle_power / cc_low_util_tokens_sec
check(
cc_total_tokens_sec.m_as(count / second) == 912,
f"114 concurrent requests × 8 tokens/s should be 912 tokens/s, got {cc_total_tokens_sec.m_as(count / second):.0f}",
cc_total_tokens_sec.to(count / second).magnitude == 912,
f"114 concurrent requests × 8 tokens/s should be 912 tokens/s, got {cc_total_tokens_sec.to(count / second).magnitude:.0f}",
)
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
cc_concurrent_str = fmt(cc_concurrent_req_value, precision=0, commas=False)
cc_tokens_req_str = fmt_rate(cc_tokens_per_sec_req_value, 'tokens/s', precision=0, commas=False)
cc_total_tokens_str = fmt_rate(round(cc_total_tokens_sec.m_as(count / second)), 'tokens/s', precision=0, commas=False)
cc_total_tokens_str = fmt_rate(round(cc_total_tokens_sec.to(count / second).magnitude), 'tokens/s', precision=0, commas=False)
cc_host_overhead_str = fmt_qty(cc_host_overhead, watt, precision=0, commas=False)
cc_total_power_str = fmt_qty(cc_total_power, watt, precision=0, commas=False)
cc_joules_token_str = fmt_qty(cc_joules_per_token, joule, precision=2, commas=False, per='token')
@@ -4202,16 +4202,16 @@ class PrecisionTradeoffCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
pt_fp32_ms_value = 2.8
pt_fp32_mem = 98 * MB
pt_fp32_mem_mb_value = pt_fp32_mem.m_as(MB)
pt_fp32_mem_mb_value = pt_fp32_mem.to(MB).magnitude
pt_fp32_acc_value = 76.13
pt_fp16_ms_value = 1.4
pt_fp16_mem = 49 * MB
pt_fp16_mem_mb_value = pt_fp16_mem.m_as(MB)
pt_fp16_mem_mb_value = pt_fp16_mem.to(MB).magnitude
pt_fp16_acc_value = 76.13
pt_fp16_util_value = 85
pt_int8_ms_value = 0.9
pt_int8_mem = 25 * MB
pt_int8_mem_mb_value = pt_int8_mem.m_as(MB)
pt_int8_mem_mb_value = pt_int8_mem.to(MB).magnitude
pt_int8_ptq_acc_value = 75.80
pt_int8_qat_acc_value = 76.05
pt_int8_util_value = 92
@@ -4578,13 +4578,13 @@ fig, ax, COLORS, plt = viz.setup_plot(figsize=(10, 5))
# ┌── 2. ARRAYS ────────────────────────────────────────────────────────────────
seq_len = np.linspace(0, 32000, 100)
layers, d_model, bytes_per_param = 80, 8192, BYTES_FP16.m_as(byte) # 70-billion-parameter model, FP16
layers, d_model, bytes_per_param = 80, 8192, BYTES_FP16.to(byte).magnitude # 70-billion-parameter model, FP16
gqa_ratio = 8 # Grouped Query Attention (8x reduction)
def get_kv_gb(batch, seq):
# KV cache size = 2 * layers * d_model * seq * batch * bytes_per_param/gqa_ratio
bytes_total = (2 * layers * d_model * seq * batch * bytes_per_param) / gqa_ratio
return (bytes_total * byte).m_as(GB)
return (bytes_total * byte).to(GB).magnitude
batches = [1, 4, 16, 32]
colors = [COLORS['BlueLine'], COLORS['GreenLine'], COLORS['OrangeLine'], COLORS['VioletLine']]
@@ -4666,9 +4666,9 @@ class LlmCaseStudyHwSpecs:
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
a100_bw = h_a100.memory.bandwidth
a100_bw_tbs_value = a100_bw.m_as(TB/second)
a100_bw_gbs_value = a100_bw.m_as(GB/second)
h100_mem_value = h_h100.memory.capacity.m_as(GiB)
a100_bw_tbs_value = a100_bw.to(TB/second).magnitude
a100_bw_gbs_value = a100_bw.to(GB/second).magnitude
h100_mem_value = h_h100.memory.capacity.to(GiB).magnitude
_serving = ServingModel().solve(
m, h_a100,
seq_len=prompt_tokens + decode_tokens,
@@ -4677,7 +4677,7 @@ class LlmCaseStudyHwSpecs:
efficiency=1.0,
)
case_study_model_weight = _serving.total_memory_required
case_study_model_weight_gb = case_study_model_weight.m_as(GB)
case_study_model_weight_gb = case_study_model_weight.to(GB).magnitude
a100_token_ms_value = case_study_model_weight_gb / a100_bw_gbs_value * THOUSAND
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -4807,11 +4807,11 @@ class LlmServingCalc:
efficiency=0.4 # realized efficiency
)
model_weight_gb_value = res.total_memory_required.m_as(GB)
model_weight_gb_value = res.total_memory_required.to(GB).magnitude
realized_tpot = res.itl
realized_tpot_ms_value = realized_tpot.m_as(ms)
realized_tpot_ms_value = realized_tpot.to(ms).magnitude
_h100_bw_gbs = h.memory.bandwidth.m_as(GB/second)
_h100_bw_gbs = h.memory.bandwidth.to(GB/second).magnitude
# Theoretical floor (100% BW efficiency)
token_time_theoretical_ms_value = (model_weight_gb_value / _h100_bw_gbs * THOUSAND)
@@ -4822,12 +4822,12 @@ class LlmServingCalc:
# KV Cache Sizing using library results
kv_cache_reserved = 72 * GB
kv_per_token = m.get_kv_cache_size(seq_len=1, batch_size=1, precision=BYTES_INT4)
kv_capacity_tokens_value = int(round((kv_cache_reserved / kv_per_token).m_as(count)))
kv_cache_gb_value = kv_cache_reserved.m_as(GB)
kv_capacity_tokens_value = int(round((kv_cache_reserved / kv_per_token).to(count).magnitude))
kv_cache_gb_value = kv_cache_reserved.to(GB).magnitude
concurrent_batch_value = int(kv_capacity_tokens_value / total_tokens)
req_time_s = res.ttft + total_decode_s
req_time_s_value = req_time_s.m_as(second)
req_time_s_value = req_time_s.to(second).magnitude
# Throughput scaling
prefill_tokens_per_s_value = 10_000 # H100 prefill rate
@@ -4844,7 +4844,7 @@ class LlmServingCalc:
remaining_vram = remaining_vram_gb_value * GB
check(
abs(decode_tokens_value * realized_tpot_ms_value / 1000 - total_decode_s.m_as(second)) < 1e-6,
abs(decode_tokens_value * realized_tpot_ms_value / 1000 - total_decode_s.to(second).magnitude) < 1e-6,
"Decode latency must equal tokens × TPOT",
)
check(
book/quarto/contents/vol1/nn_architectures/nn_architectures.qmd
+1 -1
View File
@@ -5017,7 +5017,7 @@ class ThroughputCeilingCalc:
effective_tflops_high = midrange_gpu_tflops * 0.60 # 60% utilization
headroom = effective_tflops_low * THOUSAND / sustained_gflops
objdet_sustained_tflops = (fps * objdet_gflops * GFLOPs / second).to(TFLOPs).magnitude
objdet_sustained_tflops = (fps * objdet_gflops * GFLOPs / second).to(TFLOPs / second).magnitude
objdet_headroom = effective_tflops_low/objdet_sustained_tflops
# ┌── 3. GUARD (Invariants) ──────────────────────────────────────────
book/quarto/contents/vol1/responsible_engr/responsible_engr.qmd
+27 -24
View File
@@ -1525,7 +1525,7 @@ Acknowledging that efficiency matters is the easy part; the harder engineering c
```{python}
#| label: edge-efficiency-calc
#| echo: false
from mlsysim.core.constants import watt, milliwatt, second, ms, Mparam, Kparam
from mlsysim.core.constants import watt, milliwatt, second, ms, Mparam, Kparam, param
from mlsysim.fmt import fmt, check, MarkdownStr, fmt_qty, fmt_count, fmt_time
from mlsysim import Hardware, Models
@@ -1568,8 +1568,8 @@ class EdgeEfficiencyCalc:
tiny_fits_phone = tiny_power <= smart_power and tiny_latency <= smart_latency
tiny_fits_iot = tiny_power <= iot_power and tiny_latency <= iot_latency
tiny_wear_power_margin = (wear_power / tiny_power).m_as("")
mv2_wear_power_overage = (mv2_power / wear_power).m_as("")
tiny_wear_power_margin = (wear_power / tiny_power).to("").magnitude
mv2_wear_power_overage = (mv2_power / wear_power).to("").magnitude
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(tiny_fits_iot, "TinyML should fit the IoT power and latency envelope.")
@@ -1581,25 +1581,25 @@ class EdgeEfficiencyCalc:
smart_power_str = fmt_qty(smart_power, watt, precision=0, commas=False)
smart_latency_str = fmt_qty(smart_latency, ms, precision=0, commas=False)
iot_power_str = fmt_qty(iot_power, milliwatt, precision=0, commas=False)
iot_latency_str = fmt_time(iot_latency.m_as(second), 'second', precision=0, commas=False, style='word')
iot_latency_str = fmt_time(iot_latency.to(second).magnitude, 'second', precision=0, commas=False, style='word')
cam_power_str = fmt_qty(cam_power, watt, precision=0, commas=False)
cam_latency_str = fmt_qty(cam_latency, ms, precision=0, commas=False)
wear_power_str = fmt_qty(wear_power, milliwatt, precision=0, commas=False)
wear_latency_str = fmt_qty(wear_latency, ms, precision=0, commas=False)
mv2_params_str = fmt_count(mv2_params.m_as('param'), scale='M', precision=1, commas=False)
mv2_params_str = fmt_count(mv2_params.to(param).magnitude, scale='M', precision=1, commas=False)
mv2_power_str = fmt_qty(mv2_power, watt, precision=1, commas=False)
mv2_latency_str = fmt_qty(mv2_latency, ms, precision=0, commas=False)
eff_params_str = fmt_count(eff_params.m_as('param'), scale='M', precision=1, commas=False)
eff_params_str = fmt_count(eff_params.to(param).magnitude, scale='M', precision=1, commas=False)
eff_power_str = fmt_qty(eff_power, watt, precision=1, commas=False)
eff_latency_str = fmt_qty(eff_latency, ms, precision=0, commas=False)
rn50_params_str = fmt_count(rn50_params.m_as('param'), scale='M', precision=1, commas=False)
rn50_params_str = fmt_count(rn50_params.to(param).magnitude, scale='M', precision=1, commas=False)
rn50_power_str = fmt_qty(rn50_power, watt, precision=1, commas=False)
rn50_latency_str = fmt_qty(rn50_latency, ms, precision=0, commas=False)
tiny_params_str = fmt_count(tiny_params.m_as('param'), scale='K', precision=0, commas=False)
tiny_params_str = fmt_count(tiny_params.to(param).magnitude, scale='K', precision=0, commas=False)
tiny_power_str = fmt_qty(tiny_power, milliwatt, precision=0, commas=False)
tiny_latency_str = fmt_qty(tiny_latency, ms, precision=0, commas=False)
tiny_wear_power_margin_str = fmt(tiny_wear_power_margin, precision=0, commas=False)
@@ -1675,12 +1675,13 @@ from mlsysim.core.constants import (
hour,
ms,
second,
USD,
)
class InferenceCostCalc:
"""Training vs. inference TCO comparison for a 10M-user recommendation system."""
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
gpu_rate_value = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.m_as("dollar / hour")
gpu_rate_value = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.to(USD / hour).magnitude
data_prep_hrs_value = 100 # Data preparation GPU-hours
hyperparam_hrs_value = 500 # Hyperparameter search GPU-hours
train_hrs_value = 200 # Final training GPU-hours
@@ -1689,7 +1690,7 @@ class InferenceCostCalc:
users_daily_value = 10_000_000 # Daily active users
recs_per_user_value = 20 # Recommendations per user per day
inference_ms_value = 10 # Inference latency (ms)
gpu_inf_rate_value = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.m_as("dollar / hour")
gpu_inf_rate_value = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.to(USD / hour).magnitude
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
data_prep_cost_value = data_prep_hrs_value * gpu_rate_value
@@ -1698,7 +1699,7 @@ class InferenceCostCalc:
total_train_cost_value = data_prep_cost_value + hyperparam_cost_value + train_cost_value
inferences_daily_value = users_daily_value * recs_per_user_value
gpu_seconds_daily_value = inferences_daily_value * (inference_ms_value * ms).m_as(second)
gpu_seconds_daily_value = inferences_daily_value * (inference_ms_value * ms).to(second).magnitude
gpus_needed_value = gpu_seconds_daily_value / (24 * SEC_PER_HOUR)
annual_inf_cost_value = gpus_needed_value * 24 * 365 * gpu_inf_rate_value
@@ -1773,7 +1774,7 @@ Engineers can estimate three-year total cost of ownership using a structured app
# │
# │ Note: Referenced by tco-calc cell to keep the carbon factor consistent.
# └─────────────────────────────────────────────────────────────────────────────
from mlsysim.core.constants import watt, ureg, THOUSAND
from mlsysim.core.constants import watt, ureg, THOUSAND, kg
from mlsysim import Hardware, Infrastructure
from mlsysim.fmt import fmt, check, fmt_qty
@@ -1784,12 +1785,12 @@ class CarbonFactor:
carbon_per_gpu_hr = Infrastructure.Pricing.Fleet.CarbonPerGpuHr.rate
# ┌── 2. EXECUTE (The Compute) ────────────────────────────────────────
gpu_power_kw = gpu_power.m_as(watt) / THOUSAND
carbon_per_gpu_hr_kg = carbon_per_gpu_hr.m_as(ureg.kilogram)
gpu_power_kw = gpu_power.to(watt).magnitude / THOUSAND
carbon_per_gpu_hr_kg = carbon_per_gpu_hr.to(kg).magnitude
carbon_intensity_kg_kwh = carbon_per_gpu_hr_kg / gpu_power_kw
# ┌── 3. GUARD (Invariants) ───────────────────────────────────────────
check(abs(gpu_power.m_as(watt) - 400) < 1e-9, "A100 baseline power should be 400 W.")
check(abs(gpu_power.to(watt).magnitude - 400) < 1e-9, "A100 baseline power should be 400 W.")
check(abs(carbon_per_gpu_hr_kg - 0.16) < 1e-9, "GPU-hour carbon factor should be 0.16 kg.")
# ┌── 4. OUTPUT (Formatting) ─────────────────────────────────────────────
@@ -1863,13 +1864,15 @@ from mlsysim.core.constants import (
ms,
second,
ureg,
USD,
kg,
)
class TCOCalc:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
train_gpu_rate = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.m_as("dollar / hour") # $4/hour
inf_gpu_rate = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.m_as("dollar / hour")
carbon_per_gpu_hr = Infrastructure.Pricing.Fleet.CarbonPerGpuHr.rate.m_as(ureg.kilogram)
train_gpu_rate = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.to(USD / hour).magnitude # $4/hour
inf_gpu_rate = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.to(USD / hour).magnitude
carbon_per_gpu_hr = Infrastructure.Pricing.Fleet.CarbonPerGpuHr.rate.to(kg).magnitude
t_data_prep_hrs = 100
t_hparam_exps = 50
t_hparam_cost_exp = 40.0
@@ -1878,7 +1881,7 @@ class TCOCalc:
i_users = 10_000_000
i_recs_per_user = 20
i_latency = 10 * ms
i_latency_s = i_latency.m_as(second)
i_latency_s = i_latency.to(second).magnitude
o_monitor_yr = 50000.0
o_oncall_yr = 100000.0
o_incident_yr = 20000.0
@@ -2807,14 +2810,14 @@ Teams treat responsibility as external oversight rather than engineering practic
# │ carbon_train_ratio_str, quant_reduction_pct_str,
# │ quant_savings_str, quant_carbon_str
from mlsysim import Infrastructure
from mlsysim.core.constants import DAYS_PER_YEAR, MILLION, SEC_PER_HOUR, THOUSAND, hour, ms, second, ureg
from mlsysim.core.constants import DAYS_PER_YEAR, MILLION, SEC_PER_HOUR, THOUSAND, hour, ms, second, ureg, USD, kg
from mlsysim.fmt import fmt, fmt_int, fmt_usd, check, fmt_percent
class ResponsibleTcoRecap:
# ┌── 1. LOAD (Constants) ──────────────────────────────────────────────
train_gpu_rate = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.m_as("dollar / hour")
inf_gpu_rate = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.m_as("dollar / hour")
carbon_per_gpu_hr = Infrastructure.Pricing.Fleet.CarbonPerGpuHr.rate.m_as(ureg.kilogram)
train_gpu_rate = Infrastructure.Pricing.Cloud.GpuTrainingPerHour.rate.to(USD / hour).magnitude
inf_gpu_rate = Infrastructure.Pricing.Cloud.GpuInferencePerHour.rate.to(USD / hour).magnitude
carbon_per_gpu_hr = Infrastructure.Pricing.Fleet.CarbonPerGpuHr.rate.to(kg).magnitude
t_data_prep_hrs = 100
t_hparam_exps = 50
t_hparam_cost_exp = 40.0
@@ -2822,7 +2825,7 @@ class ResponsibleTcoRecap:
t_cycles_3yr = 12
i_users = 10_000_000
i_recs_per_user = 20
i_latency_s = (10 * ms).m_as(second)
i_latency_s = (10 * ms).to(second).magnitude
o_monitor_yr = 50000.0
o_oncall_yr = 100000.0
o_incident_yr = 20000.0