cs249r_book/book/tools/scripts/quizzes/_audit/benchmarking_audit.json

{
  "metadata": {
    "chapter": "vol1/benchmarking",
    "position": 12,
    "total_chapters": 33,
    "questions_audited": 63,
    "audit_date": "2026-04-24",
    "source_file": "book/quarto/contents/vol1/benchmarking/benchmarking.qmd",
    "schema_version": 2,
    "generated_by": "quizzes/generate_quizzes.py",
    "generated_on": "2026-04-24",
    "model": "gpt-5.4",
    "total_sections": 0,
    "sections_with_quizzes": 0,
    "sections_without_quizzes": 0
  },
  "overall": {
    "quality_grade": "B",
    "summary": "This is a strong quiz set overall: questions are mostly grounded, scenario-based, and aligned with the chapter's systems focus. The main remaining issues are distributional rather than catastrophic: overuse of MCQ, a few overly easy TF/FILL items, some formulaic learning objectives, and several places where prior-chapter concepts are re-explained more than necessary."
  },
  "build_up": {
    "prior_terms_appropriately_used": [
      "goodhart's law",
      "memory wall",
      "roofline model",
      "quantization",
      "int8 quantization",
      "mixed-precision training",
      "data drift",
      "silent degradation",
      "edge deployment",
      "batch inference",
      "latency",
      "throughput",
      "data selection",
      "model compression",
      "hardware acceleration",
      "bert",
      "resnet-50",
      "mobilenetv2",
      "edge tpu",
      "all-reduce",
      "data loading",
      "model serving"
    ],
    "redefinition_violations": [],
    "missed_buildup_opportunities": [
      {
        "section_id": "#sec-benchmarking-machine-learning-benchmarking-framework-70b8",
        "question_index": 4,
        "concept": "Goodhart's Law",
        "explanation": "The question could assume reader familiarity with Goodhart's Law from earlier chapters instead of restating the full principle in generic terms. At chapter 12, using the term directly would better leverage build-up."
      },
      {
        "section_id": "#sec-benchmarking-system-benchmarking-suites-e946",
        "question_index": 1,
        "concept": "roofline reasoning / memory-bound vs compute-bound",
        "explanation": "The answer re-explains a prior-chapter hardware concept at a fairly introductory level. Since roofline and memory-bound reasoning were established earlier, the item could be tighter and more chapter-specific."
      },
      {
        "section_id": "#sec-benchmarking-training-vs-inference-evaluation-a3be",
        "question_index": 0,
        "concept": "latency, throughput, time-to-accuracy",
        "explanation": "This question is pedagogically fine, but it restates distinctions the reader has already met in earlier chapters on systems, training, and operations. It could demand more chapter-specific benchmarking reasoning instead of reintroducing the contrast at a high level."
      },
      {
        "section_id": "#sec-benchmarking-training-benchmarks-96da",
        "question_index": 2,
        "concept": "mixed precision / lower precision trade-offs",
        "explanation": "The answer spends space re-explaining a prior-chapter training optimization concept rather than pushing further into benchmark design implications. A more advanced framing could assume mixed-precision familiarity and focus on benchmark validity."
      },
      {
        "section_id": "#sec-benchmarking-model-data-benchmarking-e0ca",
        "question_index": 4,
        "concept": "calibration",
        "explanation": "The item explains calibration/quality dimensions broadly rather than assuming calibration as known prior vocabulary from workflow chapters. The chapter-specific value is still present, but the setup is a bit padded."
      }
    ],
    "forward_reference_violations": [],
    "chapter_level_assessment": "The quiz set generally leverages prior context appropriately, especially around quantization, roofline reasoning, serving latency, and Goodhart's Law. However, several questions still read as if written slightly in isolation, re-explaining prior concepts instead of assuming the chapter-12 reader already knows them."
  },
  "distribution": {
    "type_mix": {
      "MCQ": 29,
      "SHORT": 20,
      "TF": 8,
      "FILL": 5,
      "ORDER": 1
    },
    "type_mix_assessment": "over-MCQ and under-ORDER; SHORT is healthy, TF/FILL are acceptable, but the chapter is noticeably more multiple-choice-heavy than the target mix and offers too little process-order variety",
    "section_count_outliers": []
  },
  "per_question_issues": [
    {
      "section_id": "#sec-benchmarking-machine-learning-benchmarking-framework-70b8",
      "question_index": 2,
      "issue_type": "easy_tf",
      "severity": "low",
      "description": "The True/False item is directionally correct but fairly obvious from the chapter's repeated emphasis that end-to-end latency includes more than model inference. It risks testing recognition of the chapter's headline warning rather than reasoning.",
      "suggested_fix": "Make the claim more tempting by adding a concrete pipeline breakdown or a scenario where inference dominates only part of the stack, so the student must reason about relative contributions rather than reject a familiar simplification."
    },
    {
      "section_id": "#sec-benchmarking-machine-learning-benchmarking-framework-70b8",
      "question_index": 4,
      "issue_type": "trivia_fill",
      "severity": "medium",
      "description": "The FILL answer 'proxies' is explicitly phrased in the chapter and functions mostly as term recall. Although the sentence gives context, the conceptual work is minimal compared with the chapter's systems-level emphasis.",
      "suggested_fix": "Convert this into a SHORT or MCQ that asks what follows from benchmarks being proxies, such as how benchmark design should change when deployment conditions differ from evaluation conditions."
    },
    {
      "section_id": "#sec-benchmarking-historical-context-7350",
      "question_index": 3,
      "issue_type": "easy_tf",
      "severity": "low",
      "description": "The statement is too easy because the chapter repeatedly frames modern benchmarking as multi-objective rather than replacing throughput outright. The student can answer from tone more than from careful understanding.",
      "suggested_fix": "Use a more discriminating misconception, such as claiming that adding energy metrics makes performance comparisons across domains directly commensurable, then require justification grounded in deployment context."
    },
    {
      "section_id": "#sec-benchmarking-system-benchmarking-suites-e946",
      "question_index": 3,
      "issue_type": "trivia_fill",
      "severity": "low",
      "description": "The blank 'performance' in 'fallacy of peak performance' is close to phrase completion rather than systems reasoning. It names a concept but does not meaningfully test whether the student can use it.",
      "suggested_fix": "Replace with an application question asking when a peak-spec claim is least predictive and why, ideally using arithmetic intensity or workload specificity."
    },
    {
      "section_id": "#sec-benchmarking-benchmark-components-97cc",
      "question_index": 3,
      "issue_type": "trivia_fill",
      "severity": "medium",
      "description": "This FILL item effectively asks for the label 'run rules' from nearby prose. It is grounded, but it tests terminology more than inferential understanding of reproducibility design.",
      "suggested_fix": "Recast as a SHORT asking why fixed seeds alone are insufficient without execution rules, or an MCQ contrasting run rules with harnesses and system specifications in a concrete scenario."
    },
    {
      "section_id": "#sec-benchmarking-training-vs-inference-evaluation-a3be",
      "question_index": 0,
      "issue_type": "recall_only",
      "severity": "medium",
      "description": "This MCQ is correct and useful, but for chapter position 12 it is somewhat broad and introductory. It mainly checks recall of the training-vs-inference contrast rather than advanced benchmarking judgment.",
      "suggested_fix": "Anchor it in a concrete hardware or deployment scenario where the student must choose which benchmark family and which metric should govern a decision."
    },
    {
      "section_id": "#sec-benchmarking-training-vs-inference-evaluation-a3be",
      "question_index": 2,
      "issue_type": "easy_tf",
      "severity": "low",
      "description": "The True/False claim is easy to dismiss because the chapter and prior chapters repeatedly emphasize that metric choice depends on workload phase, not hardware class. It does not create much productive tension.",
      "suggested_fix": "Use a more plausible misconception, such as claiming that the same accelerator procurement decision can be made from throughput alone for both training and inference if precision is fixed."
    },
    {
      "section_id": "#sec-benchmarking-training-benchmarks-96da",
      "question_index": 1,
      "issue_type": "vague_lo",
      "severity": "low",
      "description": "The learning objective says 'Calculate and interpret strong-scaling efficiency,' but the item mostly guides the student to a qualitative interpretation after a very simple arithmetic setup. The LO slightly overstates the cognitive demand.",
      "suggested_fix": "Either make the calculation a bit less scaffolded or tighten the LO to emphasize interpretation of strong-scaling efficiency from measured runtimes."
    },
    {
      "section_id": "#sec-benchmarking-inference-benchmarks-2c1f",
      "question_index": 2,
      "issue_type": "tautological_lo",
      "severity": "low",
      "description": "The learning objective closely mirrors the stem: 'Apply Amdahl's Law to infer the limit of component-only inference optimization.' It is valid, but it reads like a restatement of the question rather than a broader outcome.",
      "suggested_fix": "Broaden the LO to something like 'Evaluate when component-level optimization meaningfully changes service-level latency' so it captures a transferable skill."
    },
    {
      "section_id": "#sec-benchmarking-inference-benchmarks-2c1f",
      "question_index": 4,
      "issue_type": "trivia_fill",
      "severity": "medium",
      "description": "The answer 'cold-start' is the obvious term from the surrounding section and the context strongly narrows the blank. This lands more as vocabulary retrieval than system-level benchmarking reasoning.",
      "suggested_fix": "Ask instead how cold-start latency changes benchmark interpretation for intermittent workloads, or compare warm-instance versus cold-start measurements in a deployment decision."
    },
    {
      "section_id": "#sec-benchmarking-power-measurement-techniques-bcc2",
      "question_index": 3,
      "issue_type": "easy_tf",
      "severity": "low",
      "description": "The False answer is fairly direct because the chapter strongly emphasizes memory movement as a dominant energy cost. The statement is more of a chapter catchphrase inversion than a nuanced misconception.",
      "suggested_fix": "Make it more subtle by contrasting two workloads, one compute-heavy and one memory-bound, and ask whether compute-only measurement would remain valid across both."
    },
    {
      "section_id": "#sec-benchmarking-benchmarking-limitations-best-practices-9d65",
      "question_index": 4,
      "issue_type": "trivia_fill",
      "severity": "low",
      "description": "The fill-in term 'lottery' is easy if the reader remembers the named phrase and does not require applying the concept. It is conceptually relevant but lighter than the rest of the section's critique-oriented material.",
      "suggested_fix": "Replace with an applied question asking how benchmark conclusions might reverse across hardware targets, or how to design evaluation to reduce hardware-lottery bias."
    },
    {
      "section_id": "#sec-benchmarking-production-considerations-084b",
      "question_index": 0,
      "issue_type": "throwaway_distractor",
      "severity": "low",
      "description": "In the traffic-burst question, 'clean labels' and 'constant arithmetic intensity' are weak distractors because they are obviously unrelated to bursty arrival patterns. This makes the item easier than it needs to be.",
      "suggested_fix": "Use stronger alternatives such as warm-system-state assumptions, queue-free serving assumptions, or batching assumptions that are genuinely plausible under changing traffic patterns."
    },
    {
      "section_id": "#sec-benchmarking-summary-5b23",
      "question_index": 2,
      "issue_type": "throwaway_distractor",
      "severity": "low",
      "description": "The distractors are somewhat uneven: 'Precision is an energy lever' and 'Benchmarks are moving targets' are clearly from different takeaway categories and are less plausible than the Amdahl choice once the student reads the stem.",
      "suggested_fix": "Use alternatives that all concern why speedups shrink in practice, such as end-to-end overheads, tail constraints, and workload mismatch, then make Amdahl's Law the most complete explanation."
    }
  ],
  "recommended_action": "minor_fixes_recommended"
}