cs249r_book/book/tools/scripts/quizzes/_audit/inference_audit.json

{
  "metadata": {
    "chapter": "vol2/inference",
    "position": 26,
    "total_chapters": 33,
    "questions_audited": 59,
    "audit_date": "2026-04-24",
    "source_file": "book/quarto/contents/vol2/inference/inference.qmd",
    "schema_version": 2,
    "generated_by": "quizzes/generate_quizzes.py",
    "generated_on": "2026-04-24",
    "model": "gpt-5.4",
    "total_sections": 0,
    "sections_with_quizzes": 0,
    "sections_without_quizzes": 0
  },
  "overall": {
    "quality_grade": "B",
    "summary": "This chapter's quiz set is strong overall: it is well grounded in the text, appropriately advanced for chapter 26/33, and generally tests systems reasoning rather than trivia. The main remaining issues are a few weak FILL items, some question-count distribution drift relative to the spec, and a handful of answers/LOs that could be tighter or more conceptually demanding."
  },
  "build_up": {
    "prior_terms_appropriately_used": [
      "tensor parallelism",
      "pipeline parallelism",
      "autoscaling",
      "load balancing",
      "tail latency",
      "queries per second (qps)",
      "kv cache",
      "model serving",
      "online inference",
      "distributed training",
      "memory bandwidth",
      "arithmetic intensity",
      "all-reduce",
      "quantization",
      "post-training quantization",
      "quantization-aware training",
      "int8",
      "fp16",
      "fp32",
      "gpu",
      "tpu",
      "moe",
      "speculative decoding",
      "little's law",
      "continuous batching"
    ],
    "redefinition_violations": [],
    "missed_buildup_opportunities": [
      {
        "section_id": "#sec-inference-scale-weight-quantization-serving-097b",
        "question_index": 1,
        "concept": "quantization / outlier-aware quantization",
        "explanation": "The chapter body itself says quantization fundamentals are prior knowledge. The question's stem and answer are fine, but the section as a whole slightly re-teaches foundational quantization framing instead of leaning harder on prior knowledge and testing deployment consequences."
      },
      {
        "section_id": "#sec-inference-scale-load-balancing-request-routing-4997",
        "question_index": 4,
        "concept": "circuit breaker",
        "explanation": "The FILL answer spends some space re-explaining the generic mechanism. Since circuit breakers are already common prior systems vocabulary for this audience, the item could assume more familiarity and focus more specifically on inference-fleet failure containment."
      }
    ],
    "forward_reference_violations": [],
    "chapter_level_assessment": "The quiz set generally leverages prior vocabulary appropriately and feels chapter-aware rather than isolated. Difficulty is appropriate for chapter 26/33: most questions are specialized, system-level, and application-oriented, though a few items could lean even more on prior knowledge instead of lightly reintroducing concepts."
  },
  "distribution": {
    "type_mix": {
      "MCQ": 29,
      "SHORT": 14,
      "TF": 9,
      "FILL": 3,
      "ORDER": 4
    },
    "type_mix_assessment": "Slightly over-MCQ and under-FILL/ORDER relative to target, but still pedagogically acceptable. Bigger issue is section-level count drift: several full-quiz sections have only 3 questions despite the spec's 4\u20136 expectation for Tier 1.",
    "section_count_outliers": [
      {
        "section_id": "#sec-inference-logic-wall",
        "count": 3,
        "expected": "4\u20136"
      },
      {
        "section_id": "#sec-inference-scale-case-studies-9689",
        "count": 3,
        "expected": "4\u20136"
      },
      {
        "section_id": "#sec-inference-scale-fallacies-pitfalls-55f9",
        "count": 3,
        "expected": "4\u20136"
      }
    ]
  },
  "per_question_issues": [
    {
      "section_id": "#sec-inference-scale-serving-framework-selection-45d3",
      "question_index": 2,
      "issue_type": "trivia_fill",
      "severity": "medium",
      "description": "This FILL item mainly tests recall of the word 'stateful.' The surrounding sentence makes the answer almost inevitable, and the concept is more important than the name; as written it behaves more like lightweight vocabulary recall than conceptual inference.",
      "suggested_fix": "Replace with a SHORT or MCQ that asks for the operational consequence of statefulness, such as sticky routing, draining on scale-down, or costly context reconstruction after failure."
    },
    {
      "section_id": "#sec-inference-logic-wall",
      "question_index": 1,
      "issue_type": "other",
      "severity": "medium",
      "description": "For a section framed around the new 'Logic Wall' concept, the quiz under-samples that exact section-specific content. Only the first item directly targets the logic-wall framing; the remaining two mostly revisit general batching/scheduling material already covered heavily in the previous section.",
      "suggested_fix": "Add a fourth question tied directly to test-time compute scaling, such as reasoning-token latency expansion, dynamic compute allocation, or the difference between throughput SLOs and reasoning SLOs."
    },
    {
      "section_id": "#sec-inference-scale-load-balancing-request-routing-4997",
      "question_index": 4,
      "issue_type": "trivia_fill",
      "severity": "medium",
      "description": "This FILL is answerable from a familiar phrase pattern rather than from inferential reasoning about the section. It edges toward naming the mechanism instead of testing why fail-fast isolation matters in inference fleets.",
      "suggested_fix": "Convert to MCQ or SHORT using a concrete overloaded-replica scenario that asks what mechanism prevents cascading timeouts and why failing fast is better than continuing to queue."
    },
    {
      "section_id": "#sec-inference-scale-autoscaling-168d",
      "question_index": 2,
      "issue_type": "missing_explanation",
      "severity": "low",
      "description": "The MCQ answer explains why the correct combined strategy is right, but it does not explicitly engage a plausible distractor in as much depth as the stronger MCQs elsewhere in the chapter. Given how tempting 'reactive only' and 'scheduled only' are, the explanation could do more contrastive work.",
      "suggested_fix": "Add one sentence explicitly noting that predictive-only misses unexpected bursts and reactive-only loses to multi-minute GPU cold starts."
    },
    {
      "section_id": "#sec-inference-scale-weight-quantization-serving-097b",
      "question_index": 1,
      "issue_type": "missing_explanation",
      "severity": "low",
      "description": "The AWQ MCQ explains the correct idea, but the distractors are not equally plausible and the explanation does not fully unpack why the 'uniform/no calibration' alternative is specifically the wrong mental model. This weakens the discrimination power a bit.",
      "suggested_fix": "Strengthen one distractor into a more realistic confusion with SmoothQuant or GPTQ, then explicitly contrast AWQ's activation-salience mechanism against that competing method."
    },
    {
      "section_id": "#sec-inference-scale-case-studies-9689",
      "question_index": 1,
      "issue_type": "vague_lo",
      "severity": "low",
      "description": "The learning objective, 'Match a production case study to the dominant serving technique it exemplifies,' is serviceable but slightly classification-oriented and undersells the synthesis value of the question. For a late-volume chapter, the LO could be sharper about comparing architectures under different bottlenecks.",
      "suggested_fix": "Revise the LO to something like 'Compare production serving architectures and infer which bottleneck drove each system's dominant optimization strategy.'"
    },
    {
      "section_id": "#sec-inference-scale-summary-bb6a",
      "question_index": 1,
      "issue_type": "tautological_lo",
      "severity": "low",
      "description": "The learning objective largely mirrors the stem: both ask the reader to explain how inference operationalizes earlier ML systems design decisions. This is not wrong, but it is a bit tautological and could name a more concrete outcome.",
      "suggested_fix": "Make the LO more testable, e.g. 'Justify why serving architecture becomes a first-class design surface once latency, reliability, and cost dominate production behavior.'"
    }
  ],
  "recommended_action": "minor_fixes_recommended"
}