Files
cs249r_book/binder/tests/test_taxonomy_coverage.py

66 lines
3.0 KiB
Python

"""Coverage numbers are computed from recorded judgments that must stay valid."""
from pathlib import Path
import pytest
import yaml
from core.taxonomy_coverage import coverage_report, write_report
REPO_ROOT = Path(__file__).resolve().parents[2]
@pytest.fixture
def repo(tmp_path):
concepts = tmp_path / "taxonomy"
(concepts / "coverage").mkdir(parents=True)
(concepts / "taxonomy.yml").symlink_to(REPO_ROOT / "taxonomy/taxonomy.yml")
(tmp_path / "books").symlink_to(REPO_ROOT / "books")
return tmp_path
def checklist(topics, held_out=True):
return {"id": "test", "title": "Test checklist", "basis": "A bounded source.", "held_out": held_out, "topics": topics}
def save(root, data):
(root / "taxonomy/coverage/test.yml").write_text(yaml.safe_dump(data))
def test_numbers_are_computed_and_weighted_by_sources(repo):
save(repo, checklist([
{"topic": "KV caching", "sources": ["a", "b", "c"], "status": "covered", "concepts": ["mlsys.kv-cache"]},
{"topic": "Paged attention", "sources": ["a"], "status": "partial", "subject": "llm-inference"},
{"topic": "Visualization", "status": "missing"},
{"topic": "ML for systems", "status": "out-of-scope"},
]))
report = write_report(repo)
summary = report["checklists"][0]["summary"]
assert summary["in_scope"] == 3
assert summary["covered_pct"] == 33.3
assert summary["homed_pct"] == 66.7
assert summary["weighted_covered_pct"] == 60.0
assert report["errors"] == []
assert (repo / "books/_build/concepts/taxonomy-coverage.md").exists()
@pytest.mark.parametrize("topic,error", [
({"topic": "Gone", "status": "covered", "concepts": ["mlsys.removed"]}, "no longer exist: mlsys.removed"),
({"topic": "No concept", "status": "covered"}, "covered requires at least one concept"),
({"topic": "No subject", "status": "partial"}, "partial requires a subject"),
({"topic": "Bad subject", "status": "partial", "subject": "Z9.9"}, "unknown subject Z9.9"),
({"topic": "Mutable code rejected", "status": "partial", "subject": "L4.4"}, "unknown subject L4.4"),
({"topic": "Stale subject", "status": "covered", "concepts": ["mlsys.kv-cache"], "subject": "request-scheduling"}, "disagrees with concept homes"),
({"topic": "Missing with concept", "status": "missing", "concepts": ["mlsys.kv-cache"]}, "missing must not cite concepts"),
({"topic": "Bad key", "status": "missing", "extra": None}, "unknown topic keys: extra"),
({"topic": "Bad status", "status": "kind of"}, "unknown status"),
])
def test_stale_or_malformed_judgments_are_reported(repo, topic, error):
save(repo, checklist([topic]))
assert any(error in e for e in coverage_report(repo)["errors"])
def test_checklist_must_declare_whether_it_is_held_out(repo):
data = checklist([{"topic": "KV caching", "status": "covered", "concepts": ["mlsys.kv-cache"]}])
del data["held_out"]
save(repo, data)
assert any("held_out" in e for e in coverage_report(repo)["errors"])