mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-09 10:39:41 -06:00
1959 lines
78 KiB
JSON
1959 lines
78 KiB
JSON
{
|
|
"schema_version": 4,
|
|
"taxonomy_version": "0.4-draft",
|
|
"evidence_status_note": "Every numeric field in this file is deterministically computed in Python from repository YAML/CSV/JSON artifacts and recorded study JSON logs.",
|
|
"historical_campaign_narrative_note": "Historical process counts narrated in taxonomy/CHANGELOG.md and taxonomy/README.md for earlier 0.2-draft, 0.3-draft, and 0.4-draft campaigns (such as intermediate reviewer proposal counts and historical diff summaries) are archival campaign narratives from those earlier commits; only metrics recomputed from repository files in this summary carry verified study-receipt status.",
|
|
"taxonomy_file_hashes": {
|
|
"baseline_dev_ml_systems_taxonomy_yml_sha256": "a3eefe42c766e031c43430aba061a56605c94117decf459b83962213f01de39e",
|
|
"genesis_0_4_baseline_json_gz_sha256": "2b49f250a20c758d4cd46b32ab37b149baf395aaae50b1d5ab7d982a21f6ff09",
|
|
"amended_ml_systems_taxonomy_yml_sha256": "d0c00b3f4a4a100bedbbee425d788fdd5d60a629ee002b347b304cf0c3ce17f8",
|
|
"non_guides_taxonomy_sha256": "fb85799731aff22ecf1f25b2c6334c1bf65efbb662af403ee1c12fae710f6f08",
|
|
"term_provenance_yml_sha256": "5780fe72efecf5d65a0de75a81c990c85837826c1a1af676a16241c8a253a0fc",
|
|
"burned_clusters_json_sha256": "cbc72fa6e769fec7aa8015fd6d154a425674e5124b379d35dd10724f4b48fef8",
|
|
"mlsys_taxonomy_concepts_csv_sha256": "0f6fd203286d47e5ce71120bfd26a4dccdf264384f581b21ec89e2d04c830083",
|
|
"mlsys_taxonomy_subjects_csv_sha256": "6ab62915e0d830bebf33013a9387483cff18152c331c7b4dba6377b7cfcfdf29",
|
|
"mlsys_taxonomy_linkml_yml_sha256": "f5dbf997760c077b013ac15880a589b83fa5f4ef203ea75950d9340842dd1340"
|
|
},
|
|
"live_counts": {
|
|
"groups": 5,
|
|
"areas": 20,
|
|
"subjects": 101,
|
|
"concepts": 878,
|
|
"relationships": 2016,
|
|
"specializes_edges": 100,
|
|
"preferred_labels_at_head": 878,
|
|
"non_preferred_entry_terms_at_dev_baseline": 1254,
|
|
"non_preferred_entry_terms_at_head": 1251,
|
|
"total_labels_and_entry_terms_at_dev_baseline": 2132,
|
|
"total_labels_and_entry_terms_at_head": 2129,
|
|
"grandfathered_draft_alias_exceptions": 7,
|
|
"grandfathered_draft_alias_exception_concepts": 7,
|
|
"grandfathered_draft_alias_exception_strings": 8,
|
|
"staffml_vault_question_count": 10716,
|
|
"per_group_counts": {
|
|
"F": {
|
|
"subjects": 12,
|
|
"primary_concepts": 94,
|
|
"primary_or_secondary_concepts": 115
|
|
},
|
|
"S": {
|
|
"subjects": 22,
|
|
"primary_concepts": 206,
|
|
"primary_or_secondary_concepts": 225
|
|
},
|
|
"P": {
|
|
"subjects": 34,
|
|
"primary_concepts": 301,
|
|
"primary_or_secondary_concepts": 336
|
|
},
|
|
"Q": {
|
|
"subjects": 23,
|
|
"primary_concepts": 200,
|
|
"primary_or_secondary_concepts": 240
|
|
},
|
|
"T": {
|
|
"subjects": 10,
|
|
"primary_concepts": 77,
|
|
"primary_or_secondary_concepts": 106
|
|
},
|
|
"T_by_area": {
|
|
"T1": {
|
|
"subjects": 5,
|
|
"primary_concepts": 25,
|
|
"primary_or_secondary_concepts": 44
|
|
},
|
|
"T2": {
|
|
"subjects": 5,
|
|
"primary_concepts": 52,
|
|
"primary_or_secondary_concepts": 64
|
|
}
|
|
}
|
|
},
|
|
"graph_connectivity": {
|
|
"connected_components": 1,
|
|
"min_degree": 2,
|
|
"max_degree": 33,
|
|
"mean_shortest_path": 5.03,
|
|
"diameter": 9
|
|
},
|
|
"warrant_csv_counts": {
|
|
"corpus_chapters_gt_0": 699,
|
|
"literary_warrant_gt_0": 471,
|
|
"organizational_warrant_gt_0": 425,
|
|
"user_warrant_proxy_gt_0": 626,
|
|
"zero_warrant_concepts": 27,
|
|
"concepts_with_references_field": 224,
|
|
"concepts_with_sources_field": 0
|
|
},
|
|
"compound_term_counts": {
|
|
"concept_labels_and_versus_vs_punct_safe": 94,
|
|
"concept_labels_and_or_versus_vs_punct_safe": 95,
|
|
"aliases_and_versus_vs_punct_safe": 16,
|
|
"total_labels_and_aliases_and_versus_vs_punct_safe": 110,
|
|
"concept_labels_legacy_vs_dot_pattern": 86
|
|
},
|
|
"historical_coverage_checklists": [
|
|
{
|
|
"id": "courses-and-standards-2026",
|
|
"file": "taxonomy/coverage/courses-and-standards-2026.yml",
|
|
"held_out": false,
|
|
"topics_total": 214,
|
|
"in_scope": 208,
|
|
"covered": 208,
|
|
"partial": 0,
|
|
"missing": 0,
|
|
"out_of_scope": 6,
|
|
"covered_pct": 100.0,
|
|
"homed_pct": 100.0
|
|
},
|
|
{
|
|
"id": "mlsys-papers-2025-2026",
|
|
"file": "taxonomy/coverage/mlsys-papers-2025-2026.yml",
|
|
"held_out": false,
|
|
"topics_total": 196,
|
|
"in_scope": 177,
|
|
"covered": 160,
|
|
"partial": 16,
|
|
"missing": 1,
|
|
"out_of_scope": 19,
|
|
"covered_pct": 90.4,
|
|
"homed_pct": 99.4
|
|
},
|
|
{
|
|
"id": "mlsysbook-vol1-vol2-primary",
|
|
"file": "taxonomy/coverage/mlsysbook-vol1-vol2-primary.yml",
|
|
"held_out": false,
|
|
"topics_total": 221,
|
|
"in_scope": 221,
|
|
"covered": 199,
|
|
"partial": 21,
|
|
"missing": 1,
|
|
"out_of_scope": 0,
|
|
"covered_pct": 90.0,
|
|
"homed_pct": 99.5
|
|
},
|
|
{
|
|
"id": "mlsysbook-vol3-vol4-primary",
|
|
"file": "taxonomy/coverage/mlsysbook-vol3-vol4-primary.yml",
|
|
"held_out": false,
|
|
"topics_total": 284,
|
|
"in_scope": 283,
|
|
"covered": 242,
|
|
"partial": 41,
|
|
"missing": 0,
|
|
"out_of_scope": 1,
|
|
"covered_pct": 85.5,
|
|
"homed_pct": 100.0
|
|
},
|
|
{
|
|
"id": "systems-venues-2025",
|
|
"file": "taxonomy/coverage/systems-venues-2025.yml",
|
|
"held_out": false,
|
|
"topics_total": 135,
|
|
"in_scope": 135,
|
|
"covered": 88,
|
|
"partial": 47,
|
|
"missing": 0,
|
|
"out_of_scope": 0,
|
|
"covered_pct": 65.2,
|
|
"homed_pct": 100.0
|
|
},
|
|
{
|
|
"id": "systems-venues-2026",
|
|
"file": "taxonomy/coverage/systems-venues-2026.yml",
|
|
"held_out": false,
|
|
"topics_total": 202,
|
|
"in_scope": 202,
|
|
"covered": 45,
|
|
"partial": 150,
|
|
"missing": 7,
|
|
"out_of_scope": 0,
|
|
"covered_pct": 22.3,
|
|
"homed_pct": 96.5
|
|
}
|
|
]
|
|
},
|
|
"staffml_300_agreement_runs": [
|
|
{
|
|
"file": "taxonomy/agreement/staffml-300-2026-10.yml",
|
|
"file_sha256": "3fc6c3a5c57b8740be81ac7a22db0f8ff87ba1b2ed51d6b2cf5ad169490625be",
|
|
"sample_id": "staffml-300-2026-10",
|
|
"run_id": "run-1",
|
|
"evaluated_taxonomy_version": "0.2-draft",
|
|
"evaluated_concept_count": 714,
|
|
"held_out": true,
|
|
"n_items": 300,
|
|
"both_non_empty": 280,
|
|
"single_empty": 12,
|
|
"both_empty": 8,
|
|
"subject_kappa_all_items": 0.781,
|
|
"conditional_hooper_pct": 65.7,
|
|
"conditional_rolling_pct": 74.6,
|
|
"all_item_hooper_pct": 64.0,
|
|
"all_item_rolling_pct": 72.3
|
|
},
|
|
{
|
|
"file": "taxonomy/agreement/staffml-300-2026-10.yml",
|
|
"file_sha256": "3fc6c3a5c57b8740be81ac7a22db0f8ff87ba1b2ed51d6b2cf5ad169490625be",
|
|
"sample_id": "staffml-300-2026-10",
|
|
"run_id": "run-2",
|
|
"evaluated_taxonomy_version": "0.2-draft",
|
|
"evaluated_concept_count": 727,
|
|
"held_out": false,
|
|
"n_items": 300,
|
|
"both_non_empty": 286,
|
|
"single_empty": 12,
|
|
"both_empty": 2,
|
|
"subject_kappa_all_items": 0.832,
|
|
"conditional_hooper_pct": 74.7,
|
|
"conditional_rolling_pct": 81.0,
|
|
"all_item_hooper_pct": 71.8,
|
|
"all_item_rolling_pct": 77.9
|
|
},
|
|
{
|
|
"file": "taxonomy/agreement/staffml-300-heldout-2026-10.yml",
|
|
"file_sha256": "d42259ff26fc58fa07f2c80dcc6a0cda6f067e6950798ba09812bc0304e6914b",
|
|
"sample_id": "staffml-300-heldout-2026-10",
|
|
"run_id": "run-1",
|
|
"evaluated_taxonomy_version": "0.2-draft",
|
|
"evaluated_concept_count": 727,
|
|
"held_out": true,
|
|
"n_items": 300,
|
|
"both_non_empty": 282,
|
|
"single_empty": 11,
|
|
"both_empty": 7,
|
|
"subject_kappa_all_items": 0.802,
|
|
"conditional_hooper_pct": 67.3,
|
|
"conditional_rolling_pct": 74.9,
|
|
"all_item_hooper_pct": 65.6,
|
|
"all_item_rolling_pct": 72.8
|
|
}
|
|
],
|
|
"staffml_300_per_track_subject_agreement": {
|
|
"run-1": {
|
|
"cloud": {
|
|
"agreed": 85,
|
|
"total": 96,
|
|
"exact_agreement_pct": 88.5
|
|
},
|
|
"edge": {
|
|
"agreed": 57,
|
|
"total": 83,
|
|
"exact_agreement_pct": 68.7
|
|
},
|
|
"global": {
|
|
"agreed": 53,
|
|
"total": 61,
|
|
"exact_agreement_pct": 86.9
|
|
},
|
|
"mobile": {
|
|
"agreed": 30,
|
|
"total": 46,
|
|
"exact_agreement_pct": 65.2
|
|
},
|
|
"tinyml": {
|
|
"agreed": 11,
|
|
"total": 14,
|
|
"exact_agreement_pct": 78.6
|
|
}
|
|
},
|
|
"run-2": {
|
|
"cloud": {
|
|
"agreed": 85,
|
|
"total": 96,
|
|
"exact_agreement_pct": 88.5
|
|
},
|
|
"edge": {
|
|
"agreed": 63,
|
|
"total": 83,
|
|
"exact_agreement_pct": 75.9
|
|
},
|
|
"global": {
|
|
"agreed": 53,
|
|
"total": 61,
|
|
"exact_agreement_pct": 86.9
|
|
},
|
|
"mobile": {
|
|
"agreed": 36,
|
|
"total": 46,
|
|
"exact_agreement_pct": 78.3
|
|
},
|
|
"tinyml": {
|
|
"agreed": 14,
|
|
"total": 14,
|
|
"exact_agreement_pct": 100.0
|
|
}
|
|
}
|
|
},
|
|
"phase1_blind_method_design": {
|
|
"pre_registration_commit": "be0837c4f3283a434009e42d74d294caacd98765",
|
|
"seed": 20261102,
|
|
"runs": {
|
|
"codex": {
|
|
"model": "gpt-6-astra",
|
|
"returncode": 0,
|
|
"output_sha256": "c564a79d1d776765f3e0e1b5589bc1adaeb6c257c721c79fdd8c4e81241d1dab",
|
|
"isolation": {
|
|
"working_directory": "/tmp/taxonomy-blind-p1-codex",
|
|
"user_instructions_disabled": true,
|
|
"memory_disabled": true,
|
|
"file_and_web_tools_disabled": false,
|
|
"mechanism": "Clean CODEX_HOME containing only auth.json (no AGENTS.md, CODEX_TASKS.md, memories, skills, or config.toml) + --skip-git-repo-check --ephemeral --ignore-user-config --ignore-rules -s read-only in empty /tmp directory"
|
|
},
|
|
"matched_repo_token_hits_in_output": 0,
|
|
"matched_repo_token_hits_in_transcript": 0
|
|
},
|
|
"claude": {
|
|
"model": "claude-opus-5-5",
|
|
"returncode": 0,
|
|
"output_sha256": "be1a9be7d5586c7da41fb5143042501c6234505b0c68e9e4634982dab60f18ab",
|
|
"isolation": {
|
|
"working_directory": "/tmp/taxonomy-blind-p1-claude",
|
|
"user_instructions_disabled": true,
|
|
"memory_disabled": true,
|
|
"file_and_web_tools_disabled": true,
|
|
"mechanism": "--safe-mode --disable-slash-commands --no-session-persistence --exclude-dynamic-system-prompt-sections --system-prompt ... --tools \"\" in empty /tmp directory outside <HOME>/GitHub"
|
|
},
|
|
"matched_repo_token_hits_in_output": 0,
|
|
"matched_repo_token_hits_in_transcript": 0
|
|
},
|
|
"gemini": {
|
|
"model": "gemini-4-argon-high",
|
|
"returncode": 0,
|
|
"output_sha256": "60f1d4278f94e32cb837794f7c226fe3c8220f0ecee076cd6d7f3f4ef344ec26",
|
|
"isolation": {
|
|
"working_directory": "/tmp/taxonomy-blind-p1-gemini",
|
|
"user_instructions_disabled": false,
|
|
"memory_disabled": true,
|
|
"file_and_web_tools_disabled": false,
|
|
"mechanism": "Run in empty /tmp directory with --mode plan --sandbox and no --add-dir; no project-level GEMINI.md or Claude auto-memory is reachable; global ~/.gemini/GEMINI.md contains only general workflow preferences and zero taxonomy terms (verified by leak scan)."
|
|
},
|
|
"matched_repo_token_hits_in_output": 0,
|
|
"matched_repo_token_hits_in_transcript": 0
|
|
}
|
|
}
|
|
},
|
|
"phase2_blind_rubric_and_citation_screen": {
|
|
"primary_standard_texts_opened": false,
|
|
"summary": {
|
|
"Method-A": {
|
|
"author": "codex (gpt-6-astra, reasoning=high)",
|
|
"per_judge_total_out_of_40": {
|
|
"judge-claude": 34.0,
|
|
"judge-codex": 38.0,
|
|
"judge-gemini": 34.0
|
|
},
|
|
"all_judges_mean_total_out_of_40": 35.333,
|
|
"all_judges_mean_per_dim_out_of_5": 4.417,
|
|
"cross_family_mean_total_out_of_40": 34.0,
|
|
"cross_family_mean_per_dim_out_of_5": 4.25,
|
|
"dim_means_all": {
|
|
"construction_steps": 4.333,
|
|
"evidence_and_warrant_rules": 4.667,
|
|
"sampling_frame": 4.667,
|
|
"representation_and_identifiers": 4.5,
|
|
"validation_and_agreement": 4.167,
|
|
"governance_and_maintenance": 4.333,
|
|
"stop_conditions_and_scope_boundaries": 4.167,
|
|
"standards_and_primary_citations": 4.5
|
|
},
|
|
"dim_means_cross": {
|
|
"construction_steps": 4.0,
|
|
"evidence_and_warrant_rules": 4.5,
|
|
"sampling_frame": 4.5,
|
|
"representation_and_identifiers": 4.25,
|
|
"validation_and_agreement": 4.0,
|
|
"governance_and_maintenance": 4.25,
|
|
"stop_conditions_and_scope_boundaries": 4.0,
|
|
"standards_and_primary_citations": 4.5
|
|
}
|
|
},
|
|
"Method-B": {
|
|
"author": "claude (claude-opus-5-5)",
|
|
"per_judge_total_out_of_40": {
|
|
"judge-claude": 35.0,
|
|
"judge-codex": 34.5,
|
|
"judge-gemini": 40.0
|
|
},
|
|
"all_judges_mean_total_out_of_40": 36.5,
|
|
"all_judges_mean_per_dim_out_of_5": 4.562,
|
|
"cross_family_mean_total_out_of_40": 37.25,
|
|
"cross_family_mean_per_dim_out_of_5": 4.656,
|
|
"dim_means_all": {
|
|
"construction_steps": 4.667,
|
|
"evidence_and_warrant_rules": 4.333,
|
|
"sampling_frame": 4.667,
|
|
"representation_and_identifiers": 4.667,
|
|
"validation_and_agreement": 4.667,
|
|
"governance_and_maintenance": 4.667,
|
|
"stop_conditions_and_scope_boundaries": 4.667,
|
|
"standards_and_primary_citations": 4.167
|
|
},
|
|
"dim_means_cross": {
|
|
"construction_steps": 4.75,
|
|
"evidence_and_warrant_rules": 4.5,
|
|
"sampling_frame": 4.75,
|
|
"representation_and_identifiers": 4.5,
|
|
"validation_and_agreement": 4.75,
|
|
"governance_and_maintenance": 4.75,
|
|
"stop_conditions_and_scope_boundaries": 4.75,
|
|
"standards_and_primary_citations": 4.5
|
|
}
|
|
},
|
|
"Method-C": {
|
|
"author": "gemini (gemini-4-argon-high)",
|
|
"per_judge_total_out_of_40": {
|
|
"judge-claude": 26.0,
|
|
"judge-codex": 25.5,
|
|
"judge-gemini": 38.0
|
|
},
|
|
"all_judges_mean_total_out_of_40": 29.833,
|
|
"all_judges_mean_per_dim_out_of_5": 3.729,
|
|
"cross_family_mean_total_out_of_40": 25.75,
|
|
"cross_family_mean_per_dim_out_of_5": 3.219,
|
|
"dim_means_all": {
|
|
"construction_steps": 3.833,
|
|
"evidence_and_warrant_rules": 3.5,
|
|
"sampling_frame": 4.0,
|
|
"representation_and_identifiers": 3.833,
|
|
"validation_and_agreement": 3.667,
|
|
"governance_and_maintenance": 3.667,
|
|
"stop_conditions_and_scope_boundaries": 4.0,
|
|
"standards_and_primary_citations": 3.333
|
|
},
|
|
"dim_means_cross": {
|
|
"construction_steps": 3.5,
|
|
"evidence_and_warrant_rules": 3.0,
|
|
"sampling_frame": 3.75,
|
|
"representation_and_identifiers": 3.25,
|
|
"validation_and_agreement": 3.0,
|
|
"governance_and_maintenance": 3.0,
|
|
"stop_conditions_and_scope_boundaries": 3.5,
|
|
"standards_and_primary_citations": 2.75
|
|
}
|
|
}
|
|
},
|
|
"reference_knowledge_citation_screen": "No primary sources were opened; verdicts rest on the supplied local check and reference knowledge, so upheld means consistent with reference knowledge, and ISO pinpoints remain unconfirmed. Grouped rows state how many citations they cover. Totals: Method-A 36 checked (31 upheld, 5 downgraded, 0 vacated); Method-B 112 checked (108 upheld, 4 downgraded, 0 vacated); Method-C 79 checked (45 upheld, 33 downgraded, 1 vacated). The local check is not a valid clause audit because it mixes in internal section numbers, ISO clauses and the standard's own name. Method-A is the most reliable. Method-B has a strong bibliography but cites standards by topic and does not apply its provenance tags. Method-C needs correction before adoption, especially its nonexistent SKOS properties, wrong Cimino numbers, squared MASI distance, wrong Z39.19 topics, conflated MLPerf citation, and unconfirmed single-child and IEEE P3116 claims."
|
|
},
|
|
"phase3_preregistered_sampling_split": {
|
|
"preregistered_date": "2026-11-02",
|
|
"random_seed": 20261103,
|
|
"artifact_hashes": {
|
|
"dev_sources_sha256": "0849d619afb0e745fdde9f5fef715d67cd3d49da31459e2a0b502dafab866df0",
|
|
"held_out_slice_sha256": "221fb94438db2d5e0f78f4dbb73c99c081e73d8788c89cbd12468f6a054607c7",
|
|
"total_units": 382,
|
|
"dev_units": 200,
|
|
"held_out_test_units": 126,
|
|
"reserve_units": 56
|
|
},
|
|
"total_unique_clusters_pre_deduplication": 97,
|
|
"total_clusters_under_phase3_2_merges": 95,
|
|
"total_unique_clusters_deduplicated": 88,
|
|
"canonical_internal_clusters_in_382": 39,
|
|
"canonical_external_clusters_in_382": 49,
|
|
"total_author_deduplicated_lineage_components": 41,
|
|
"external_author_deduplicated_lineage_components": 40,
|
|
"historical_phase3_units_by_initial_phase3_split": {
|
|
"dev": 200,
|
|
"held_out_test": 126,
|
|
"reserve": 56
|
|
},
|
|
"historical_phase3_units_by_current_exposure_split": {
|
|
"dev": 382,
|
|
"held_out_test": 0,
|
|
"reserve": 0
|
|
},
|
|
"raw_cross_split_clusters_at_phase3": 0,
|
|
"canonical_cross_split_clusters_under_full_10_merges": [
|
|
"pf_cov_mlsys_25_26_mlsys2026",
|
|
"pf_stanford_cs336",
|
|
"pf_ut_cs378"
|
|
],
|
|
"clusters_by_split": {
|
|
"dev": {
|
|
"total_clusters_raw": 76,
|
|
"pct_of_97": 78.4,
|
|
"P4_only": 47,
|
|
"Q4_only": 23,
|
|
"shared_P4_Q4": 6,
|
|
"external_clusters_raw": 44,
|
|
"internal_clusters_raw": 32
|
|
},
|
|
"held_out_test": {
|
|
"total_clusters_raw": 15,
|
|
"pct_of_97": 15.5,
|
|
"P4_only": 8,
|
|
"Q4_only": 6,
|
|
"shared_P4_Q4": 1,
|
|
"external_clusters_raw": 10,
|
|
"internal_clusters_raw": 5
|
|
},
|
|
"reserve": {
|
|
"total_clusters_raw": 6,
|
|
"pct_of_97": 6.2,
|
|
"P4_only": 3,
|
|
"Q4_only": 3,
|
|
"shared_P4_Q4": 0,
|
|
"external_clusters_raw": 4,
|
|
"internal_clusters_raw": 2
|
|
}
|
|
},
|
|
"units_by_split_area_origin": {
|
|
"dev": {
|
|
"total_units": 200,
|
|
"P4_total": 142,
|
|
"P4_external": 94,
|
|
"P4_internal": 48,
|
|
"Q4_total": 58,
|
|
"Q4_external": 14,
|
|
"Q4_internal": 44
|
|
},
|
|
"held_out_test": {
|
|
"total_units": 126,
|
|
"P4_total": 116,
|
|
"P4_external": 75,
|
|
"P4_internal": 41,
|
|
"Q4_total": 10,
|
|
"Q4_external": 4,
|
|
"Q4_internal": 6
|
|
},
|
|
"reserve": {
|
|
"total_units": 56,
|
|
"P4_total": 47,
|
|
"P4_external": 46,
|
|
"P4_internal": 1,
|
|
"Q4_total": 9,
|
|
"Q4_external": 2,
|
|
"Q4_internal": 7
|
|
}
|
|
},
|
|
"post_phase6_campaign_manifest_state": {
|
|
"total_units": 382,
|
|
"total_deduplicated_clusters": 88,
|
|
"total_author_deduplicated_lineage_components": 41,
|
|
"external_author_deduplicated_lineage_components": 40,
|
|
"dev_units": 382,
|
|
"dev_clusters": 88,
|
|
"held_out_test_units": 0,
|
|
"held_out_test_clusters": 0,
|
|
"reserve_units": 0,
|
|
"reserve_clusters": 0,
|
|
"reserve_cluster_ids": []
|
|
}
|
|
},
|
|
"phase4_and_phase5_rebuild_and_audit": {
|
|
"phase4_builder_counts": {
|
|
"claude": {
|
|
"family": "claude",
|
|
"p4_concepts": 66,
|
|
"q4_concepts": 32,
|
|
"relationships": 75,
|
|
"sha256": "e4666d02a01c645915af851889b272a71629440007ae067af10d3dd8e3c66503"
|
|
},
|
|
"codex": {
|
|
"family": "codex",
|
|
"p4_concepts": 54,
|
|
"q4_concepts": 24,
|
|
"relationships": 112,
|
|
"sha256": "3c3bb155e903f665843b023530e03d13366689ee3806e7bd38e8ea8b6e481b24"
|
|
},
|
|
"gemini": {
|
|
"family": "gemini",
|
|
"p4_concepts": 51,
|
|
"q4_concepts": 26,
|
|
"relationships": 85,
|
|
"sha256": "3aa3dfe901320b5b2ca1a7651516be1731da470e121ee63802005067b502c3c1"
|
|
}
|
|
},
|
|
"phase4_reconciled_counts": {
|
|
"total_admitted_concepts": 111,
|
|
"p4_admitted_concepts": 68,
|
|
"q4_admitted_concepts": 43,
|
|
"multi_builder_concepts": 63,
|
|
"single_builder_multi_stratum_concepts": 48,
|
|
"admitted_relationships": 178,
|
|
"rejected_candidates": 27
|
|
},
|
|
"phase5_reproducibility": {
|
|
"before_sha256": "9468a3dfff0a3c19b433e9c38fa034c994d5fbb64da28bda2f9dd11b8be646ed",
|
|
"after_sha256": "9468a3dfff0a3c19b433e9c38fa034c994d5fbb64da28bda2f9dd11b8be646ed",
|
|
"deterministic_rerun_match": true
|
|
},
|
|
"phase5_anti_gaming_checks": {
|
|
"total_concepts_checked": 111,
|
|
"product_or_sku_leaks": 0,
|
|
"product_flags": [],
|
|
"split_quantity_metric_x_workload_flags": 0,
|
|
"metric_workload_flags": [],
|
|
"shallow_padding_definitions": 0,
|
|
"padding_flags": []
|
|
},
|
|
"phase5_curated_summary_quote_audit": {
|
|
"total_rebuilt_concepts": 111,
|
|
"passage_kind_of_phase4_prose_units": "curated_summary",
|
|
"concepts_with_curated_summary_substring_match": 111,
|
|
"concepts_with_curated_summary_word_bounded_match": 111,
|
|
"total_warrant_citations_across_111_concepts": 282,
|
|
"warrant_citations_curated_summary_substring_match": 282,
|
|
"warrant_citations_curated_summary_word_bounded_match": 282,
|
|
"curated_summary_caveat": "Phase 4 prose source cards were author-curated topical summaries rather than verbatim primary-document excerpts; for example, P4-S3-SGLANG-2024 compressed SGLang's 'constrained decoding' (arXiv:2312.07104) as 'grammar-constrained decoding', and P4-S2-CMU11868-02 summarized grammar/JSON-schema mask compilation as 'compiling JSON schemas or context-free grammars into token-masking finite-state automata' (whereas unrestricted CFGs require pushdown/Earley-style stacks rather than pure finite-state automata)."
|
|
},
|
|
"phase5_citation_audit_sample": {
|
|
"sampled_concepts_count": 55,
|
|
"total_citations_audited": 137,
|
|
"upheld": 137,
|
|
"downgraded": 0,
|
|
"vacated": 0,
|
|
"sample_verdicts": 40
|
|
},
|
|
"phase5_independent_semantic_audit": {
|
|
"claude": {
|
|
"checked": 35,
|
|
"upheld": 26,
|
|
"downgraded": 8,
|
|
"vacated": 1,
|
|
"findings": 9
|
|
},
|
|
"codex": {
|
|
"checked": 35,
|
|
"upheld": 27,
|
|
"downgraded": 6,
|
|
"vacated": 2,
|
|
"findings": 8
|
|
}
|
|
}
|
|
},
|
|
"phase6_held_out_comparison": {
|
|
"unit_accounting": {
|
|
"total_held_out_test_units": 126,
|
|
"excluded_internal_s1_book_units": 7,
|
|
"cov_external_units_evaluated": 79,
|
|
"cov_external_clusters_raw": 10,
|
|
"cov_external_lineages_deduplicated": 9,
|
|
"cov_course_id_overlap_with_dev_unit_ids": [
|
|
"P4-S2-STANFORD-CS336-01"
|
|
],
|
|
"cov_course_or_author_overlap_with_dev_unit_ids": [
|
|
"P4-S2-STANFORD-CS336-01",
|
|
"P4-S3-SLORA-2024",
|
|
"Q4-S2-BERKELEY-CS294-267-01"
|
|
],
|
|
"cov_courses_and_standards_checklist_design_input_unit_ids": [
|
|
"P4-COV-courses-and-standards-2026-212",
|
|
"Q4-COV-courses-and-standards-2026-191"
|
|
],
|
|
"cov_exposed_to_0_4_draft_design_inputs_unit_ids": [
|
|
"P4-COV-courses-and-standards-2026-212",
|
|
"P4-S2-STANFORD-CS336-01",
|
|
"P4-S3-SLORA-2024",
|
|
"Q4-COV-courses-and-standards-2026-191",
|
|
"Q4-S2-BERKELEY-CS294-267-01"
|
|
],
|
|
"cov_exposed_of_other_6_non_systems_venues_2026": 5,
|
|
"cov_systems_venues_2026_units": 73,
|
|
"cov_systems_venues_2026_share_pct": 92.4,
|
|
"cov_systems_venues_2026_author_metadata_note": "taxonomy/coverage/systems-venues-2026.yml records accepted paper titles and venue codes (OSDI, NSDI, EuroSys, ASPLOS 2026) without full per-paper author rosters; author-overlap filtering on those 73 paper-title units could not be verified retrospectively, whereas prospective campaigns require single-work external clusters with all listed authors (or institutional_author=true with institutional_organization).",
|
|
"cov_out_of_scope_dropped": 0,
|
|
"idx_units_evaluated": 43,
|
|
"idx_internal_s4_staffml_units": 40,
|
|
"idx_external_q4_units_shared_with_cov": 3,
|
|
"idx_both_non_empty": 43,
|
|
"idx_single_empty": 0,
|
|
"idx_both_empty": 0
|
|
},
|
|
"results_by_arm": {
|
|
"rebuilt_synthesized_method": {
|
|
"arm": "Vocab-X",
|
|
"true_identity": "rebuilt_synthesized_method",
|
|
"vocabulary_size": {
|
|
"total_concepts": 111,
|
|
"p4_concepts": 68,
|
|
"q4_concepts": 43
|
|
},
|
|
"held_out_coverage": {
|
|
"held_out_units_evaluated": 79,
|
|
"consensus_covered_pct": 10.1,
|
|
"any_judge_covered_pct": 36.7,
|
|
"p4_consensus_covered_pct": 10.7,
|
|
"q4_consensus_covered_pct": 0.0,
|
|
"consensus_exact_single_concept_pct": 2.5,
|
|
"mean_expressibility_score_pct": 17.4,
|
|
"judge_binary_kappa": 0.325
|
|
},
|
|
"held_out_indexing_agreement": {
|
|
"held_out_items_indexed": 43,
|
|
"subject_exact_pct": 93.0,
|
|
"subject_kappa": 0.917,
|
|
"hooper_consistency_pct": 72.1,
|
|
"rolling_consistency_pct": 81.2,
|
|
"alpha_masi": 0.539
|
|
},
|
|
"anti_gaming_and_burden": {
|
|
"max_single_concept_share_pct": 6.94,
|
|
"distinct_concepts_used_in_indexing": 55,
|
|
"top_3_most_frequent_slugs": [
|
|
[
|
|
"device-resource-envelope",
|
|
15
|
|
],
|
|
[
|
|
"kv-cache",
|
|
11
|
|
],
|
|
[
|
|
"input-output-guardrail",
|
|
8
|
|
]
|
|
]
|
|
},
|
|
"seeded_gap_audit": {
|
|
"seeded_removals_k": 10,
|
|
"removed_slugs": [
|
|
"batch-prediction",
|
|
"compound-ai-system",
|
|
"kv-cache",
|
|
"on-device-inference",
|
|
"parallel-candidate-search",
|
|
"post-training-quantization",
|
|
"prefill-decode-co-execution",
|
|
"prefill-decode-disaggregation",
|
|
"request-scheduling",
|
|
"speculative-decoding"
|
|
],
|
|
"affected_judgments": 29,
|
|
"detected_degradations": 16,
|
|
"seeded_gap_detection_pct": 55.2
|
|
}
|
|
},
|
|
"published_0_4_draft": {
|
|
"arm": "Vocab-Y",
|
|
"true_identity": "published_0_4_draft",
|
|
"vocabulary_size": {
|
|
"total_concepts": 125,
|
|
"p4_concepts": 95,
|
|
"q4_concepts": 30
|
|
},
|
|
"held_out_coverage": {
|
|
"held_out_units_evaluated": 79,
|
|
"consensus_covered_pct": 36.7,
|
|
"any_judge_covered_pct": 72.2,
|
|
"p4_consensus_covered_pct": 38.7,
|
|
"q4_consensus_covered_pct": 0.0,
|
|
"consensus_exact_single_concept_pct": 19.0,
|
|
"mean_expressibility_score_pct": 42.4,
|
|
"judge_binary_kappa": 0.366
|
|
},
|
|
"held_out_indexing_agreement": {
|
|
"held_out_items_indexed": 43,
|
|
"subject_exact_pct": 97.7,
|
|
"subject_kappa": 0.971,
|
|
"hooper_consistency_pct": 81.8,
|
|
"rolling_consistency_pct": 87.6,
|
|
"alpha_masi": 0.724
|
|
},
|
|
"anti_gaming_and_burden": {
|
|
"max_single_concept_share_pct": 9.84,
|
|
"distinct_concepts_used_in_indexing": 44,
|
|
"top_3_most_frequent_slugs": [
|
|
[
|
|
"on-device-runtime",
|
|
18
|
|
],
|
|
[
|
|
"batching",
|
|
9
|
|
],
|
|
[
|
|
"kv-cache",
|
|
9
|
|
]
|
|
]
|
|
},
|
|
"seeded_gap_audit": {
|
|
"seeded_removals_k": 10,
|
|
"removed_slugs": [
|
|
"admission",
|
|
"autoscaling",
|
|
"batch-inference",
|
|
"cpu-gpu-hybrid-inference",
|
|
"fidelity-adaptive-serving",
|
|
"kv-cache",
|
|
"kv-cache-offload",
|
|
"model-swapping",
|
|
"on-device-runtime",
|
|
"ota-model-update"
|
|
],
|
|
"affected_judgments": 40,
|
|
"detected_degradations": 24,
|
|
"seeded_gap_detection_pct": 60.0
|
|
}
|
|
}
|
|
},
|
|
"phase6_leaf_specificity": {
|
|
"vocab_y": {
|
|
"total_concepts_in_arm": 125,
|
|
"leaf_concepts_in_arm": 116,
|
|
"non_leaf_parent_concepts_in_arm": 9,
|
|
"total_tag_assignments": 183,
|
|
"leaf_tag_assignments": 148,
|
|
"leaf_assignment_pct": 80.9,
|
|
"distinct_concepts_assigned": 44,
|
|
"vocabulary_utilization_pct": 35.2,
|
|
"max_single_concept_share_pct": 9.84
|
|
},
|
|
"vocab_x": {
|
|
"total_concepts_in_arm": 111,
|
|
"leaf_concepts_in_arm": 94,
|
|
"non_leaf_parent_concepts_in_arm": 17,
|
|
"total_tag_assignments": 216,
|
|
"leaf_tag_assignments": 186,
|
|
"leaf_assignment_pct": 86.1,
|
|
"distinct_concepts_assigned": 55,
|
|
"vocabulary_utilization_pct": 49.5,
|
|
"max_single_concept_share_pct": 6.94
|
|
}
|
|
},
|
|
"staffml_40_heldout_trace": {
|
|
"total_s4_heldout_units": 40,
|
|
"from_staffml_300_heldout_2026_10": 38,
|
|
"from_staffml_300_2026_10": 2,
|
|
"first_sample_question_ids": [
|
|
"edge-1247",
|
|
"tinyml-1536"
|
|
],
|
|
"in_proposed_concepts_or_subject_only_or_adjudication": 0,
|
|
"in_review_queue_total": 13,
|
|
"in_review_queue_from_38_heldout": 12,
|
|
"in_review_queue_from_2_first_sample": 1,
|
|
"review_queue_question_ids": [
|
|
"cloud-0089",
|
|
"cloud-1246",
|
|
"cloud-3372",
|
|
"cloud-3803",
|
|
"cloud-4147",
|
|
"cloud-4305",
|
|
"edge-1247",
|
|
"edge-1807",
|
|
"edge-1842",
|
|
"mobile-1147",
|
|
"mobile-1424",
|
|
"mobile-2053",
|
|
"tinyml-0673"
|
|
]
|
|
},
|
|
"coverage_by_subset": {
|
|
"all_79_external": {
|
|
"n_units": 79,
|
|
"vocab_y_consensus_expressible": 29,
|
|
"vocab_y_consensus_expressible_pct": 36.7,
|
|
"vocab_y_consensus_exact_single_concept": 15,
|
|
"vocab_y_consensus_exact_single_concept_pct": 19.0,
|
|
"vocab_y_any_judge_covered": 57,
|
|
"vocab_y_any_judge_pct": 72.2,
|
|
"vocab_x_consensus_expressible": 8,
|
|
"vocab_x_consensus_expressible_pct": 10.1,
|
|
"vocab_x_consensus_exact_single_concept": 2,
|
|
"vocab_x_consensus_exact_single_concept_pct": 2.5,
|
|
"vocab_x_any_judge_covered": 29,
|
|
"vocab_x_any_judge_pct": 36.7
|
|
},
|
|
"course_id_disjoint_78_external": {
|
|
"n_units": 78,
|
|
"vocab_y_consensus_expressible": 29,
|
|
"vocab_y_consensus_expressible_pct": 37.2,
|
|
"vocab_y_consensus_exact_single_concept": 15,
|
|
"vocab_y_consensus_exact_single_concept_pct": 19.2,
|
|
"vocab_y_any_judge_covered": 57,
|
|
"vocab_y_any_judge_pct": 73.1,
|
|
"vocab_x_consensus_expressible": 8,
|
|
"vocab_x_consensus_expressible_pct": 10.3,
|
|
"vocab_x_consensus_exact_single_concept": 2,
|
|
"vocab_x_consensus_exact_single_concept_pct": 2.6,
|
|
"vocab_x_any_judge_covered": 29,
|
|
"vocab_x_any_judge_pct": 37.2
|
|
},
|
|
"dev_disjoint_76_external": {
|
|
"n_units": 76,
|
|
"vocab_y_consensus_expressible": 28,
|
|
"vocab_y_consensus_expressible_pct": 36.8,
|
|
"vocab_y_consensus_exact_single_concept": 14,
|
|
"vocab_y_consensus_exact_single_concept_pct": 18.4,
|
|
"vocab_y_any_judge_covered": 56,
|
|
"vocab_y_any_judge_pct": 73.7,
|
|
"vocab_x_consensus_expressible": 8,
|
|
"vocab_x_consensus_expressible_pct": 10.5,
|
|
"vocab_x_consensus_exact_single_concept": 2,
|
|
"vocab_x_consensus_exact_single_concept_pct": 2.6,
|
|
"vocab_x_any_judge_covered": 29,
|
|
"vocab_x_any_judge_pct": 38.2
|
|
},
|
|
"unexposed_73_systems_venues_2026": {
|
|
"n_units": 73,
|
|
"vocab_y_consensus_expressible": 28,
|
|
"vocab_y_consensus_expressible_pct": 38.4,
|
|
"vocab_y_consensus_exact_single_concept": 14,
|
|
"vocab_y_consensus_exact_single_concept_pct": 19.2,
|
|
"vocab_y_any_judge_covered": 55,
|
|
"vocab_y_any_judge_pct": 75.3,
|
|
"vocab_x_consensus_expressible": 8,
|
|
"vocab_x_consensus_expressible_pct": 11.0,
|
|
"vocab_x_consensus_exact_single_concept": 2,
|
|
"vocab_x_consensus_exact_single_concept_pct": 2.7,
|
|
"vocab_x_any_judge_covered": 28,
|
|
"vocab_x_any_judge_pct": 38.4
|
|
},
|
|
"other_6_external_s2_s3_s5": {
|
|
"n_units": 6,
|
|
"vocab_y_consensus_expressible": 1,
|
|
"vocab_y_consensus_expressible_pct": 16.7,
|
|
"vocab_y_consensus_exact_single_concept": 1,
|
|
"vocab_y_consensus_exact_single_concept_pct": 16.7,
|
|
"vocab_y_any_judge_covered": 2,
|
|
"vocab_y_any_judge_pct": 33.3,
|
|
"vocab_x_consensus_expressible": 0,
|
|
"vocab_x_consensus_expressible_pct": 0.0,
|
|
"vocab_x_consensus_exact_single_concept": 0,
|
|
"vocab_x_consensus_exact_single_concept_pct": 0.0,
|
|
"vocab_x_any_judge_covered": 1,
|
|
"vocab_x_any_judge_pct": 16.7
|
|
},
|
|
"p4_75_external": {
|
|
"n_units": 75,
|
|
"vocab_y_consensus_expressible": 29,
|
|
"vocab_y_consensus_expressible_pct": 38.7,
|
|
"vocab_y_consensus_exact_single_concept": 15,
|
|
"vocab_y_consensus_exact_single_concept_pct": 20.0,
|
|
"vocab_y_any_judge_covered": 57,
|
|
"vocab_y_any_judge_pct": 76.0,
|
|
"vocab_x_consensus_expressible": 8,
|
|
"vocab_x_consensus_expressible_pct": 10.7,
|
|
"vocab_x_consensus_exact_single_concept": 2,
|
|
"vocab_x_consensus_exact_single_concept_pct": 2.7,
|
|
"vocab_x_any_judge_covered": 29,
|
|
"vocab_x_any_judge_pct": 38.7
|
|
},
|
|
"q4_4_external": {
|
|
"n_units": 4,
|
|
"vocab_y_consensus_expressible": 0,
|
|
"vocab_y_consensus_expressible_pct": 0.0,
|
|
"vocab_y_consensus_exact_single_concept": 0,
|
|
"vocab_y_consensus_exact_single_concept_pct": 0.0,
|
|
"vocab_y_any_judge_covered": 0,
|
|
"vocab_y_any_judge_pct": 0.0,
|
|
"vocab_x_consensus_expressible": 0,
|
|
"vocab_x_consensus_expressible_pct": 0.0,
|
|
"vocab_x_consensus_exact_single_concept": 0,
|
|
"vocab_x_consensus_exact_single_concept_pct": 0.0,
|
|
"vocab_x_any_judge_covered": 0,
|
|
"vocab_x_any_judge_pct": 0.0
|
|
}
|
|
},
|
|
"consensus_coverage_diff_vocab_x_minus_vocab_y": {
|
|
"consensus_expressible_point_estimate_pp": -26.6,
|
|
"consensus_exact_single_concept_point_estimate_pp": -16.5,
|
|
"unexposed_73_expressible_point_estimate_pp": -27.4,
|
|
"unexposed_73_exact_single_concept_point_estimate_pp": -16.4,
|
|
"bootstrap_seed": 20261107,
|
|
"resamples": 10000,
|
|
"unit_bootstrap_95_ci_pp": [
|
|
-36.7,
|
|
-16.5
|
|
],
|
|
"raw_10_cluster_bootstrap_95_ci_pp": [
|
|
-44.9,
|
|
-6.7
|
|
],
|
|
"deduplicated_9_lineage_bootstrap_95_ci_pp": [
|
|
-45.1,
|
|
-6.8
|
|
],
|
|
"deduplicated_9_lineage_exact_single_concept_bootstrap_95_ci_pp": [
|
|
-31.7,
|
|
-2.6
|
|
],
|
|
"unexposed_73_unit_bootstrap_95_ci_pp": [
|
|
-38.4,
|
|
-16.4
|
|
],
|
|
"unexposed_73_exact_single_concept_unit_bootstrap_95_ci_pp": [
|
|
-26.0,
|
|
-6.8
|
|
]
|
|
},
|
|
"indexing_by_subset": {
|
|
"all_43": {
|
|
"vocab_y": {
|
|
"n_items": 43,
|
|
"both_non_empty": 43,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.971,
|
|
"hooper_pct": 81.8,
|
|
"hooper_pct_4dp": 81.7829,
|
|
"rolling_pct": 87.6,
|
|
"rolling_pct_4dp": 87.5969,
|
|
"D_o_6dp": 0.267442,
|
|
"D_e_6dp": 0.969953,
|
|
"alpha_masi": 0.724,
|
|
"alpha_masi_4dp": 0.7243,
|
|
"alpha_masi_6dp": 0.724273
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 43,
|
|
"both_non_empty": 43,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.917,
|
|
"hooper_pct": 72.1,
|
|
"hooper_pct_4dp": 72.093,
|
|
"rolling_pct": 81.2,
|
|
"rolling_pct_4dp": 81.2403,
|
|
"D_o_6dp": 0.449612,
|
|
"D_e_6dp": 0.975331,
|
|
"alpha_masi": 0.539,
|
|
"alpha_masi_4dp": 0.539,
|
|
"alpha_masi_6dp": 0.539015
|
|
}
|
|
},
|
|
"internal_40_staffml_s4": {
|
|
"vocab_y": {
|
|
"n_items": 40,
|
|
"both_non_empty": 40,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.968,
|
|
"hooper_pct": 81.7,
|
|
"hooper_pct_4dp": 81.6667,
|
|
"rolling_pct": 87.5,
|
|
"rolling_pct_4dp": 87.5,
|
|
"D_o_6dp": 0.266667,
|
|
"D_e_6dp": 0.967247,
|
|
"alpha_masi": 0.724,
|
|
"alpha_masi_4dp": 0.7243,
|
|
"alpha_masi_6dp": 0.724303
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 40,
|
|
"both_non_empty": 40,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.909,
|
|
"hooper_pct": 75.0,
|
|
"hooper_pct_4dp": 75.0,
|
|
"rolling_pct": 84.0,
|
|
"rolling_pct_4dp": 84.0,
|
|
"D_o_6dp": 0.416667,
|
|
"D_e_6dp": 0.972078,
|
|
"alpha_masi": 0.571,
|
|
"alpha_masi_4dp": 0.5714,
|
|
"alpha_masi_6dp": 0.571365
|
|
}
|
|
},
|
|
"tagging_rule_holdout_38_staffml_2026_10": {
|
|
"vocab_y": {
|
|
"n_items": 38,
|
|
"both_non_empty": 38,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.966,
|
|
"hooper_pct": 82.5,
|
|
"hooper_pct_4dp": 82.4561,
|
|
"rolling_pct": 88.2,
|
|
"rolling_pct_4dp": 88.1579,
|
|
"D_o_6dp": 0.25731,
|
|
"D_e_6dp": 0.96563,
|
|
"alpha_masi": 0.734,
|
|
"alpha_masi_4dp": 0.7335,
|
|
"alpha_masi_6dp": 0.733531
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 38,
|
|
"both_non_empty": 38,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 0.902,
|
|
"hooper_pct": 75.0,
|
|
"hooper_pct_4dp": 75.0,
|
|
"rolling_pct": 84.0,
|
|
"rolling_pct_4dp": 84.0351,
|
|
"D_o_6dp": 0.416667,
|
|
"D_e_6dp": 0.970222,
|
|
"alpha_masi": 0.571,
|
|
"alpha_masi_4dp": 0.5705,
|
|
"alpha_masi_6dp": 0.570545
|
|
}
|
|
},
|
|
"sample1_2_items_excluded_from_38_holdout": {
|
|
"unit_ids": [
|
|
"Q4-S4-SAMPLE1-Q-160",
|
|
"Q4-S4-SAMPLE1-Q-249"
|
|
],
|
|
"vocab_y": {
|
|
"n_items": 2,
|
|
"both_non_empty": 2,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 66.7,
|
|
"hooper_pct_4dp": 66.6667,
|
|
"rolling_pct": 75.0,
|
|
"rolling_pct_4dp": 75.0,
|
|
"D_o_6dp": 0.444444,
|
|
"D_e_6dp": 0.814815,
|
|
"alpha_masi": 0.455,
|
|
"alpha_masi_4dp": 0.4545,
|
|
"alpha_masi_6dp": 0.454545
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 2,
|
|
"both_non_empty": 2,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 75.0,
|
|
"hooper_pct_4dp": 75.0,
|
|
"rolling_pct": 83.3,
|
|
"rolling_pct_4dp": 83.3333,
|
|
"D_o_6dp": 0.416667,
|
|
"D_e_6dp": 0.805556,
|
|
"alpha_masi": 0.483,
|
|
"alpha_masi_4dp": 0.4828,
|
|
"alpha_masi_6dp": 0.482759
|
|
}
|
|
},
|
|
"external_3_q4_s2_s5": {
|
|
"vocab_y": {
|
|
"n_items": 3,
|
|
"both_non_empty": 3,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 83.3,
|
|
"hooper_pct_4dp": 83.3333,
|
|
"rolling_pct": 88.9,
|
|
"rolling_pct_4dp": 88.8889,
|
|
"D_o_6dp": 0.277778,
|
|
"D_e_6dp": 0.705926,
|
|
"alpha_masi": 0.607,
|
|
"alpha_masi_4dp": 0.6065,
|
|
"alpha_masi_6dp": 0.606506
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 3,
|
|
"both_non_empty": 3,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 33.3,
|
|
"hooper_pct_4dp": 33.3333,
|
|
"rolling_pct": 44.4,
|
|
"rolling_pct_4dp": 44.4444,
|
|
"D_o_6dp": 0.888889,
|
|
"D_e_6dp": 0.96,
|
|
"alpha_masi": 0.074,
|
|
"alpha_masi_4dp": 0.0741,
|
|
"alpha_masi_6dp": 0.074074
|
|
}
|
|
},
|
|
"external_2_q4_dev_disjoint": {
|
|
"unit_ids": [
|
|
"Q4-COV-courses-and-standards-2026-191",
|
|
"Q4-S5-NIST-RMF-600-1"
|
|
],
|
|
"vocab_y": {
|
|
"n_items": 2,
|
|
"both_non_empty": 2,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 100.0,
|
|
"hooper_pct_4dp": 100.0,
|
|
"rolling_pct": 100.0,
|
|
"rolling_pct_4dp": 100.0,
|
|
"D_o_6dp": 0.0,
|
|
"D_e_6dp": 0.518519,
|
|
"alpha_masi": 1.0,
|
|
"alpha_masi_4dp": 1.0,
|
|
"alpha_masi_6dp": 1.0
|
|
},
|
|
"vocab_x": {
|
|
"n_items": 2,
|
|
"both_non_empty": 2,
|
|
"single_empty": 0,
|
|
"both_empty": 0,
|
|
"subject_kappa": 1.0,
|
|
"hooper_pct": 25.0,
|
|
"hooper_pct_4dp": 25.0,
|
|
"rolling_pct": 33.3,
|
|
"rolling_pct_4dp": 33.3333,
|
|
"D_o_6dp": 0.916667,
|
|
"D_e_6dp": 0.972222,
|
|
"alpha_masi": 0.057,
|
|
"alpha_masi_4dp": 0.0571,
|
|
"alpha_masi_6dp": 0.057143
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"phase7_repeatability_pilot_q4_1": {
|
|
"input_q4_dev_units": 58,
|
|
"gold_q4_1_concepts": 7,
|
|
"round1_before_clarification": {
|
|
"per_builder_vs_gold": {
|
|
"claude": {
|
|
"proposed_count": 6,
|
|
"gold_count": 7,
|
|
"matched_gold_count": 5,
|
|
"recall_pct": 71.4,
|
|
"precision_pct": 83.3,
|
|
"hooper_vs_gold_pct": 62.5,
|
|
"rolling_vs_gold_pct": 76.9,
|
|
"matched_gold_ids": [
|
|
"mlsys.fmea",
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"missed_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.stpa"
|
|
],
|
|
"unmatched_proposed_slugs": [
|
|
"safety-constraint"
|
|
]
|
|
},
|
|
"codex": {
|
|
"proposed_count": 3,
|
|
"gold_count": 7,
|
|
"matched_gold_count": 3,
|
|
"recall_pct": 42.9,
|
|
"precision_pct": 100.0,
|
|
"hooper_vs_gold_pct": 42.9,
|
|
"rolling_vs_gold_pct": 60.0,
|
|
"matched_gold_ids": [
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"missed_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.fmea",
|
|
"mlsys.operating-domain",
|
|
"mlsys.stpa"
|
|
],
|
|
"unmatched_proposed_slugs": []
|
|
}
|
|
},
|
|
"inter_builder_agreement": {
|
|
"shared_canonical_concepts": 3,
|
|
"claude_canonical_count": 5,
|
|
"codex_canonical_count": 3,
|
|
"hooper_consistency_pct": 60.0,
|
|
"rolling_consistency_pct": 75.0,
|
|
"consensus_matched_gold_ids": [
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"union_matched_gold_ids": [
|
|
"mlsys.fmea",
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"consensus_recall_pct": 42.9,
|
|
"union_recall_pct": 71.4
|
|
}
|
|
},
|
|
"round2_post_clarification_calibration": {
|
|
"per_builder_vs_gold": {
|
|
"claude": {
|
|
"proposed_active_count": 3,
|
|
"declared_gaps_count": 4,
|
|
"gold_count": 7,
|
|
"matched_active_gold_count": 3,
|
|
"matched_active_plus_gaps_gold_count": 7,
|
|
"active_recall_pct": 42.9,
|
|
"active_plus_gaps_recall_pct": 100.0,
|
|
"active_precision_pct": 100.0,
|
|
"active_plus_gaps_precision_pct": 100.0,
|
|
"hooper_active_vs_gold_pct": 42.9,
|
|
"rolling_active_vs_gold_pct": 60.0,
|
|
"hooper_active_plus_gaps_vs_gold_pct": 100.0,
|
|
"rolling_active_plus_gaps_vs_gold_pct": 100.0,
|
|
"matched_active_gold_ids": [
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"matched_gap_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.fmea",
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.stpa"
|
|
],
|
|
"missed_gold_ids": [],
|
|
"unmatched_proposed_slugs": []
|
|
},
|
|
"codex": {
|
|
"proposed_active_count": 3,
|
|
"declared_gaps_count": 4,
|
|
"gold_count": 7,
|
|
"matched_active_gold_count": 3,
|
|
"matched_active_plus_gaps_gold_count": 7,
|
|
"active_recall_pct": 42.9,
|
|
"active_plus_gaps_recall_pct": 100.0,
|
|
"active_precision_pct": 100.0,
|
|
"active_plus_gaps_precision_pct": 100.0,
|
|
"hooper_active_vs_gold_pct": 42.9,
|
|
"rolling_active_vs_gold_pct": 60.0,
|
|
"hooper_active_plus_gaps_vs_gold_pct": 100.0,
|
|
"rolling_active_plus_gaps_vs_gold_pct": 100.0,
|
|
"matched_active_gold_ids": [
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain"
|
|
],
|
|
"matched_gap_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.fmea",
|
|
"mlsys.stpa",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"missed_gold_ids": [],
|
|
"unmatched_proposed_slugs": []
|
|
}
|
|
},
|
|
"inter_builder_agreement": {
|
|
"active_shared_canonical_concepts": 2,
|
|
"active_hooper_consistency_pct": 50.0,
|
|
"active_rolling_consistency_pct": 66.7,
|
|
"active_plus_gaps_shared_canonical_concepts": 7,
|
|
"active_plus_gaps_hooper_consistency_pct": 100.0,
|
|
"active_plus_gaps_rolling_consistency_pct": 100.0,
|
|
"consensus_active_plus_gaps_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.fmea",
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain",
|
|
"mlsys.stpa",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"union_active_plus_gaps_gold_ids": [
|
|
"mlsys.fault-tree-analysis",
|
|
"mlsys.fmea",
|
|
"mlsys.functional-insufficiency",
|
|
"mlsys.hazard-analysis",
|
|
"mlsys.operating-domain",
|
|
"mlsys.stpa",
|
|
"mlsys.tolerable-risk"
|
|
],
|
|
"consensus_active_plus_gaps_recall_pct": 100.0,
|
|
"union_active_plus_gaps_recall_pct": 100.0
|
|
}
|
|
}
|
|
},
|
|
"phase7_bucket_a_amendment": {
|
|
"updated_concepts": [
|
|
{
|
|
"id": "mlsys.benchmark",
|
|
"label": "Benchmark protocol",
|
|
"home": "system-benchmarking",
|
|
"before_definition": "A reproducible specification of workload, measurement boundary, run rules, and comparison procedure that lets results from different systems be compared and lets a system be judged against a baseline.",
|
|
"after_definition": "A reproducible specification of workload, measurement boundary, run rules, and comparison procedure that lets results from different systems be compared and lets a system be judged against a baseline.",
|
|
"before_aliases": [
|
|
"MLPerf",
|
|
"benchmark suite"
|
|
],
|
|
"after_aliases": []
|
|
},
|
|
{
|
|
"id": "mlsys.online-experiment",
|
|
"label": "Online experimentation",
|
|
"home": "evaluation-protocols",
|
|
"before_definition": "A controlled intervention used to estimate effects on deployed-system outcomes.",
|
|
"after_definition": "A controlled intervention used to estimate effects on deployed-system outcomes.",
|
|
"before_aliases": [
|
|
"A/B testing",
|
|
"A/B test",
|
|
"controlled online experiment",
|
|
"preregistered analysis plan"
|
|
],
|
|
"after_aliases": [
|
|
"A/B testing",
|
|
"A/B test",
|
|
"controlled online experiment"
|
|
]
|
|
},
|
|
{
|
|
"id": "mlsys.selective-prediction",
|
|
"label": "Selective prediction",
|
|
"home": "metrics-uncertainty",
|
|
"before_definition": "Predicting on admitted cases and abstaining or deferring the rest.",
|
|
"after_definition": "Decision-gating mechanism that emits model outputs only for admitted cases and abstains on or defers the remaining cases.",
|
|
"before_aliases": [],
|
|
"after_aliases": []
|
|
},
|
|
{
|
|
"id": "mlsys.constrained-decoding",
|
|
"label": "Constrained decoding",
|
|
"home": "structured-generation",
|
|
"before_definition": "Masking disallowed tokens at each step, typically from a grammar or schema automaton, so output satisfies a structural constraint.",
|
|
"after_definition": "Generation-control mechanism that masks disallowed tokens at each step, typically via a grammar or schema automaton, so generated output conforms to a required syntax or schema.",
|
|
"before_aliases": [
|
|
"grammar-guided decoding"
|
|
],
|
|
"after_aliases": [
|
|
"grammar-guided decoding"
|
|
]
|
|
}
|
|
],
|
|
"incident_relationships_on_redefined_concepts": {
|
|
"mlsys.constrained-decoding": [
|
|
{
|
|
"id": "rel.constrained-decoding.grammar",
|
|
"source": "mlsys.constrained-decoding",
|
|
"target": "mlsys.grammar",
|
|
"predicate": "enforces",
|
|
"conditions": "Holds when the constraint compiles to an automaton aligned with the tokenizer's vocabulary.",
|
|
"references": [
|
|
"guided-generation"
|
|
],
|
|
"kind": "mechanism",
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "implements"
|
|
},
|
|
{
|
|
"id": "rel.output-validation.constrained-decoding",
|
|
"source": "mlsys.output-validation",
|
|
"target": "mlsys.constrained-decoding",
|
|
"predicate": "is the post-hoc alternative to",
|
|
"conditions": "Validation catches semantic rule violations decoding cannot express, at the cost of retries.",
|
|
"references": [
|
|
"guided-generation"
|
|
],
|
|
"kind": "tradeoff",
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "alternative-to"
|
|
},
|
|
{
|
|
"id": "rel.constrained-decoding.function-calling",
|
|
"source": "mlsys.constrained-decoding",
|
|
"target": "mlsys.function-calling",
|
|
"relation": "enables",
|
|
"predicate": "guarantees well-formed arguments for",
|
|
"conditions": "Holds when the tool schema is compiled to a grammar.",
|
|
"references": [
|
|
"guided-generation"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "mechanism",
|
|
"reference_role": "reading-basis-not-entailment-proof"
|
|
}
|
|
],
|
|
"mlsys.selective-prediction": [
|
|
{
|
|
"id": "rel.calibration.selective-prediction",
|
|
"source": "mlsys.calibration",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "supports",
|
|
"conditions": "Abstention thresholds still require risk and coverage evaluation under the deployment distribution.",
|
|
"references": [
|
|
"calibration"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "mechanism",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "enables"
|
|
},
|
|
{
|
|
"id": "rel.conformal-prediction.selective-prediction",
|
|
"source": "mlsys.conformal-prediction",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "provides coverage-calibrated sets for",
|
|
"conditions": "Coverage holds marginally under exchangeability between calibration and test data, not under distribution shift.",
|
|
"references": [
|
|
"conformal-intro"
|
|
],
|
|
"kind": "mechanism",
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "enables"
|
|
},
|
|
{
|
|
"id": "rel.model-cascade.selective-prediction",
|
|
"source": "mlsys.model-cascade",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "relies on",
|
|
"conditions": "Each stage must abstain and defer when confidence is low; poorly calibrated scores misroute requests.",
|
|
"references": [
|
|
"frugalgpt"
|
|
],
|
|
"kind": "dependency",
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "requires"
|
|
},
|
|
{
|
|
"id": "rel.cost-sensitive-decision.selective-prediction",
|
|
"source": "mlsys.cost-sensitive-decision",
|
|
"target": "mlsys.selective-prediction",
|
|
"relation": "determines",
|
|
"predicate": "sets the abstention threshold in",
|
|
"conditions": "Deferring to a human or fallback has a known cost that can be compared with the expected cost of each error type.",
|
|
"references": [
|
|
"metrics",
|
|
"fairmlbook"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "mechanism",
|
|
"reference_role": "reading-basis-not-entailment-proof"
|
|
},
|
|
{
|
|
"id": "rel.ood-detection.selective-prediction",
|
|
"source": "mlsys.ood-detection",
|
|
"target": "mlsys.selective-prediction",
|
|
"relation": "enables",
|
|
"kind": "mechanism",
|
|
"predicate": "supplies abstention signals to",
|
|
"conditions": "The system may defer uncertain inputs.",
|
|
"references": [
|
|
"failing-loudly"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"reference_role": "reading-basis-not-entailment-proof"
|
|
},
|
|
{
|
|
"id": "rel.predictive-uncertainty.selective-prediction",
|
|
"source": "mlsys.predictive-uncertainty",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "supplies the confidence signal that decides abstention in",
|
|
"conditions": "When the estimate ranks errors well enough that abstaining on the least certain inputs lowers risk at the chosen coverage.",
|
|
"references": [
|
|
"metrics",
|
|
"selection-bias",
|
|
"calibration"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "mechanism",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "enables"
|
|
},
|
|
{
|
|
"id": "rel.semi-supervised-learning.selective-prediction",
|
|
"source": "mlsys.semi-supervised-learning",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "keeps only confident pseudo-labels through",
|
|
"conditions": "When a confidence threshold decides which unlabeled examples enter training.",
|
|
"references": [
|
|
"cmu",
|
|
"cs336",
|
|
"random-search",
|
|
"hyperband"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "dependency",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "uses"
|
|
},
|
|
{
|
|
"id": "rel.human-oversight.selective-prediction",
|
|
"source": "mlsys.human-oversight",
|
|
"target": "mlsys.selective-prediction",
|
|
"predicate": "routes abstained cases to people through",
|
|
"conditions": "When the system abstains on low-confidence inputs and a human review path exists with enough time and context to decide.",
|
|
"references": [
|
|
"aml",
|
|
"dp",
|
|
"cs329s"
|
|
],
|
|
"basis": "editorial-synthesis",
|
|
"status": "draft",
|
|
"kind": "dependency",
|
|
"reference_role": "reading-basis-not-entailment-proof",
|
|
"relation": "uses"
|
|
}
|
|
]
|
|
},
|
|
"baseline_and_head_mlsys_stpa": {
|
|
"id": "mlsys.stpa",
|
|
"label": "System-theoretic process analysis",
|
|
"added_in": "0.2-draft",
|
|
"baseline_aliases": [
|
|
"STPA"
|
|
],
|
|
"head_aliases": [
|
|
"STPA"
|
|
],
|
|
"note": "mlsys.stpa has only alias ['STPA'] in both 0.4-draft baseline and amended HEAD; 'unsafe control action' and 'UCA' were rejected Phase 4 rebuild candidates and were never present on mlsys.stpa."
|
|
},
|
|
"withdrawn_candidate_aliases": [
|
|
{
|
|
"concept_id": "mlsys.constrained-decoding",
|
|
"candidate_alias": "grammar-constrained decoding",
|
|
"reason": "Withdrawn during primary-source verification: Zheng et al., SGLang (NeurIPS 2024, arXiv:2312.07104) uses 'constrained decoding' rather than 'grammar-constrained decoding'; the phrase appeared only in the author-curated Phase 4 summary card P4-S3-SGLANG-2024 (passage_kind='curated_summary')."
|
|
}
|
|
],
|
|
"final_patch": {
|
|
"concepts_update": [
|
|
{
|
|
"id": "mlsys.constrained-decoding",
|
|
"definition": "Generation-control mechanism that masks disallowed tokens at each step, typically via a grammar or schema automaton, so generated output conforms to a required syntax or schema.",
|
|
"definition_change_kind": "editorial-clarification",
|
|
"checked_incident_relationships": [
|
|
"rel.constrained-decoding.function-calling",
|
|
"rel.constrained-decoding.grammar",
|
|
"rel.output-validation.constrained-decoding"
|
|
],
|
|
"edge_impact_note": "Checked against the baseline definition ('Masking disallowed tokens at each step, typically from a grammar or schema automaton, so output satisfies a structural constraint.') and all 3 incident relationships (rel.constrained-decoding.function-calling, rel.constrained-decoding.grammar, and rel.output-validation.constrained-decoding): adds a genus, replaces label-echoing 'structural constraint' with 'required syntax or schema' in line with the structured-generation subject definition, and retains per-step token masking and typical grammar or schema automaton guidance without shifting extension. V1 edge queue note: rel.constrained-decoding.function-calling ('guarantees well-formed arguments for ... when the tool schema is compiled to a grammar') cannot be edited in non-structural mode without holdout evaluation and is queued for V1 relationships_update to qualify that per-step valid-prefix masking guarantees well-formed arguments only for completed, non-truncated generations under sound schema-to-grammar compilation."
|
|
},
|
|
{
|
|
"id": "mlsys.selective-prediction",
|
|
"definition": "Decision-gating mechanism that emits model outputs only for admitted cases and abstains on or defers the remaining cases.",
|
|
"definition_change_kind": "editorial-clarification",
|
|
"checked_incident_relationships": [
|
|
"rel.calibration.selective-prediction",
|
|
"rel.conformal-prediction.selective-prediction",
|
|
"rel.cost-sensitive-decision.selective-prediction",
|
|
"rel.human-oversight.selective-prediction",
|
|
"rel.model-cascade.selective-prediction",
|
|
"rel.ood-detection.selective-prediction",
|
|
"rel.predictive-uncertainty.selective-prediction",
|
|
"rel.semi-supervised-learning.selective-prediction"
|
|
],
|
|
"edge_impact_note": "Checked against the baseline definition ('Predicting on admitted cases and abstaining or deferring the rest.') and all 8 incident relationships on mlsys.selective-prediction (rel.calibration.selective-prediction, rel.conformal-prediction.selective-prediction, rel.cost-sensitive-decision.selective-prediction, rel.human-oversight.selective-prediction, rel.model-cascade.selective-prediction, rel.ood-detection.selective-prediction, rel.predictive-uncertainty.selective-prediction, and rel.semi-supervised-learning.selective-prediction): adds a genus and replaces label-echoing 'Predicting' with emitting model outputs while keeping baseline 'admitted cases' and abstain-or-defer semantics intact across runtime requests, cascade deferral, human escalation, and confidence-filtered pseudo-labeling. V1 edge queue note: rel.semi-supervised-learning.selective-prediction carries extraneous baseline references ('random-search', 'hyperband') queued for V1 relationships_update cleanup."
|
|
},
|
|
{
|
|
"id": "mlsys.benchmark",
|
|
"remove_aliases": [
|
|
"MLPerf",
|
|
"benchmark suite"
|
|
],
|
|
"remove_aliases_rationales": {
|
|
"MLPerf": "Removes named multi-benchmark suite family 'MLPerf' (maintained by MLCommons; excluded from aliases under Gate 3 project scope policy inspired by Z39.19 section 6.3.3, clause pinpoint per Phase 2 outline pending V1 primary-text verification).",
|
|
"benchmark suite": "Removes multi-protocol collection 'benchmark suite', which denotes a collection of benchmarks rather than an equivalent entry term for a single reproducible benchmark protocol (Z39.19 section 8.2, clause pinpoint per Phase 2 outline pending V1 primary-text verification)."
|
|
}
|
|
},
|
|
{
|
|
"id": "mlsys.online-experiment",
|
|
"remove_aliases": [
|
|
"preregistered analysis plan"
|
|
],
|
|
"remove_aliases_rationales": {
|
|
"preregistered analysis plan": "Removes planning/governance document 'preregistered analysis plan', which is a specification artifact used by an experiment rather than an equivalence term for Online experimentation (Z39.19 section 8.2, clause pinpoint per Phase 2 outline pending V1 primary-text verification)."
|
|
}
|
|
}
|
|
],
|
|
"subject_updates": [
|
|
{
|
|
"row": "on-device-deployment",
|
|
"add_secondary_concepts": [
|
|
"mlsys.memory-planning"
|
|
],
|
|
"rationale": "Curated dev summary P4-S2-MIT65940-01 (source stratum S2, MIT 6.5940) states that 'Deploying models on edge SoCs, mobile NPUs, and microcontrollers requires ... static arena memory planning to avoid heap fragmentation under kilobyte SRAM ceilings'. Editorial scope note: static arena memory planning is an applicable on-device deployment technique for predictable allocation and fragmentation avoidance—chiefly required on bare-metal microcontrollers operating under kilobyte SRAM ceilings rather than a universal requirement across all edge SoCs and mobile NPUs; because mlsys.memory-planning is homed in subject S2.4 (ml-compilers) in area S2, it is linked here as a secondary concept rather than re-homed."
|
|
}
|
|
]
|
|
},
|
|
"r2_review_file_hashes": {
|
|
"claude-r2.yaml": "9dca9628d44b1b4a170e6f7b2f3cf6860c59a941edc625f4700b6b91fe10c1b0",
|
|
"codex-r2.yaml": "3a1c346b06233e264bdce6633ab5ae8a2277e6df0766115f20674f9a8a07940f",
|
|
"gemini-r2.yaml": "a8b3057655bf02e087b6083ef3d52f72a5e11c7c7360e6a640be0447bcc5713a"
|
|
},
|
|
"overrides": {
|
|
"codex:update:mlsys.constrained-decoding": "Two independent reviewer families (claude-opus-5-5 and Gemini 4 Argon) accepted patch.yaml as-is with extension_unchanged=true; codex (gpt-6) likewise confirmed extension_unchanged=true and kept the identical definition, definition_change_kind, and checked_incident_relationships, suggesting only an additional V1 edge-queue note on rel.output-validation.constrained-decoding inside edge_impact_note (recorded in METHOD.md and CHANGELOG.md for the next structural relationships_update pass while retaining the 2/3-accepted patch.yaml payload)."
|
|
},
|
|
"reconcile_receipt": {
|
|
"schema_version": 4,
|
|
"mode": "full-3family-reconcile",
|
|
"campaign_id": "bucket-a-campaign:fa62ec6fecdf",
|
|
"amendment_mode": "non-structural",
|
|
"allow_pre_evaluation_apply": false,
|
|
"bootstrap_provenance": true,
|
|
"is_structural": false,
|
|
"evaluation_gate_completed": false,
|
|
"unseal_holdout_receipt_sha256": null,
|
|
"evaluation_receipt_sha256": null,
|
|
"audit_receipt_sha256": null,
|
|
"newly_burned_holdout_clusters": [],
|
|
"reconcile_script_sha256": "99792ca93bf9ffeaa8245ca91205935411640683476e0b683e9649aaca9171a9",
|
|
"apply_script_sha256": "f63c143ab2512cbac8ad6907601c380f33f91ca2a5e8dc4c052b504c79aab186",
|
|
"taxonomy_schema_script_sha256": "d0b38790bc8883a0f8772959bf1581d170c06a222e360e1f42169d5a7347c9aa",
|
|
"run_reviews_script_sha256": "e4af4a1ff9506628869ade9a15584bef30e160e1c0937c4c921c8993197159cb",
|
|
"stage_reviewer_workspace_script_sha256": "2b75c3e4a4c5814b4a6ceaac858fa0959bbe3ae30905a84886c4052c93ee7a42",
|
|
"baseline_taxonomy_sha256": "a3eefe42c766e031c43430aba061a56605c94117decf459b83962213f01de39e",
|
|
"baseline_non_guides_taxonomy_sha256": "1fd44b93987fea8ec835414c04cb8417cf127ae5d0ff4fbfb7fb9ade6ba934cc",
|
|
"baseline_term_provenance_sha256": null,
|
|
"baseline_linkml_schema_sha256": "f5dbf997760c077b013ac15880a589b83fa5f4ef203ea75950d9340842dd1340",
|
|
"burned_clusters_sha256": "cbc72fa6e769fec7aa8015fd6d154a425674e5124b379d35dd10724f4b48fef8",
|
|
"patch_yaml_sha256": "fa62ec6fecdf227fc1ac4c766bcdd16bb3d09f6a5e4d1a1c4c7a9d091442b76e",
|
|
"final_yaml_sha256": "fa62ec6fecdf227fc1ac4c766bcdd16bb3d09f6a5e4d1a1c4c7a9d091442b76e",
|
|
"final_yaml_canonical_json_sha256": "6e2b950c7d0932d4eab34c0393d99e6720591dfa0a97aa0ce132b24a2169778d",
|
|
"r2_reviews_sha256": {
|
|
"claude-r2.yaml": "9dca9628d44b1b4a170e6f7b2f3cf6860c59a941edc625f4700b6b91fe10c1b0",
|
|
"codex-r2.yaml": "3a1c346b06233e264bdce6633ab5ae8a2277e6df0766115f20674f9a8a07940f",
|
|
"gemini-r2.yaml": "a8b3057655bf02e087b6083ef3d52f72a5e11c7c7360e6a640be0447bcc5713a"
|
|
},
|
|
"overrides_file": "overrides.yaml",
|
|
"overrides_sha256": "88a94dbfe42e780a6703fc786bd5f59720ac5fe80f0c12ab15379fcd72d81b37",
|
|
"sampling_manifest_sha256": "a10146f86302f1d5c53493585278da89f17369ea9e5ba9091d6877dc8f84e52e",
|
|
"workspace_isolation_r2_sha256": "04a3e6e2357ca7c5e778133882e3809735d34c9ad66a8d7f0678ce3374d0a17f",
|
|
"run_reviews_r2_sha256": "db0d72b32ee5d0f6eea442611c8a7148373979f9f61dbdea3d2f8ac3a6d9764a",
|
|
"operations_reconciled": [
|
|
"row:on-device-deployment",
|
|
"update:mlsys.benchmark",
|
|
"update:mlsys.constrained-decoding",
|
|
"update:mlsys.online-experiment",
|
|
"update:mlsys.selective-prediction"
|
|
],
|
|
"operations_applied": [
|
|
"row:on-device-deployment",
|
|
"update:mlsys.benchmark",
|
|
"update:mlsys.constrained-decoding",
|
|
"update:mlsys.online-experiment",
|
|
"update:mlsys.selective-prediction"
|
|
],
|
|
"counts_by_family": {
|
|
"claude": {
|
|
"accept": 5,
|
|
"fix": 0,
|
|
"reject": 0,
|
|
"unexplained": 0
|
|
},
|
|
"codex": {
|
|
"accept": 4,
|
|
"fix": 1,
|
|
"reject": 0,
|
|
"unexplained": 0
|
|
},
|
|
"gemini": {
|
|
"accept": 5,
|
|
"fix": 0,
|
|
"reject": 0,
|
|
"unexplained": 0
|
|
}
|
|
},
|
|
"red_flag_violations": 0
|
|
},
|
|
"apply_receipt": {
|
|
"schema_version": 2,
|
|
"campaign_id": "bucket-a-campaign:fa62ec6fecdf",
|
|
"baseline_taxonomy_sha256": "a3eefe42c766e031c43430aba061a56605c94117decf459b83962213f01de39e",
|
|
"post_apply_taxonomy_sha256": "d0c00b3f4a4a100bedbbee425d788fdd5d60a629ee002b347b304cf0c3ce17f8",
|
|
"baseline_non_guides_taxonomy_sha256": "1fd44b93987fea8ec835414c04cb8417cf127ae5d0ff4fbfb7fb9ade6ba934cc",
|
|
"post_non_guides_taxonomy_sha256": "fb85799731aff22ecf1f25b2c6334c1bf65efbb662af403ee1c12fae710f6f08",
|
|
"baseline_term_provenance_sha256": null,
|
|
"post_apply_term_provenance_sha256": "5780fe72efecf5d65a0de75a81c990c85837826c1a1af676a16241c8a253a0fc",
|
|
"sidecar_content_sha256": "c13b968d5cb5f9e65ffa85c45c7dc506a48c1fe296f0ad164cefe63581281074",
|
|
"campaign_entry_sha256": "932d98a221b0e2a81eceb3c59170f586459da9beafbb4db052d3818924043851"
|
|
},
|
|
"workspace_isolation_receipt": {
|
|
"schema_version": 3,
|
|
"mode": "crosscheck",
|
|
"isolation_scope": "staged_external_tmp_directory_outside_repo",
|
|
"stage_reviewer_workspace_script_sha256": "2b75c3e4a4c5814b4a6ceaac858fa0959bbe3ae30905a84886c4052c93ee7a42",
|
|
"baseline_taxonomy_sha256": "a3eefe42c766e031c43430aba061a56605c94117decf459b83962213f01de39e",
|
|
"included_dev_checklists": [
|
|
"courses-and-standards-2026.yml",
|
|
"mlsys-papers-2025-2026.yml",
|
|
"mlsysbook-vol1-vol2-primary.yml",
|
|
"mlsysbook-vol3-vol4-primary.yml",
|
|
"systems-venues-2025.yml",
|
|
"systems-venues-2026.yml"
|
|
],
|
|
"staged_dev_checklist_sha256": {
|
|
"courses-and-standards-2026.yml": "81770922e938e391ecc0fbc20d5d36ade00254fa94591effd479e5cc7914238f",
|
|
"mlsys-papers-2025-2026.yml": "717ccb883637844704bb68e4cca5db95ed4a8b41ce754664dc00b4a7e85ca560",
|
|
"mlsysbook-vol1-vol2-primary.yml": "901677ebce9cf47f484bebdb9d94e6589868c2441773218d849f1680e38a48c5",
|
|
"mlsysbook-vol3-vol4-primary.yml": "9cfe031a1c2b4babd87c65285a48eb4c4c69f1768c2d62c79914883c0f1aca79",
|
|
"systems-venues-2025.yml": "c498e52d3a7fe336e344e97c12536cd236ea45c21f42ad08d23cfe935aec3fe7",
|
|
"systems-venues-2026.yml": "c1407084a65d933ebdd64a19ed859a8914bc597ae7968fa1a4612247e5e60415"
|
|
},
|
|
"excluded_held_out_checklists": [],
|
|
"held_out_checklists_staged": 0,
|
|
"agreement_files_staged": 0,
|
|
"staged_dev_manifest_units": 382,
|
|
"excluded_non_dev_manifest_units": 0,
|
|
"non_dev_manifest_units_staged": 0,
|
|
"non_dev_leak_scan_token_count": 561,
|
|
"non_dev_leak_scan_tokens_by_category": {
|
|
"non_dev_manifest_tokens": 0,
|
|
"non_dev_agreement_suite_ids": 1,
|
|
"non_dev_agreement_item_ids": 560,
|
|
"held_out_checklist_tokens": 0
|
|
},
|
|
"copied_book_chapters": 145,
|
|
"excluded_non_dev_book_chapters": [],
|
|
"staged_staffml_leads": "staffml/vault/data/taxonomy-tagging-dev-leads.yaml",
|
|
"staged_staffml_leads_sha256": "8ba7c24e5649185714405642282333a6be01df06dac82447cc72b706f47cfd4c",
|
|
"contextual_discovery_only_not_gate2_or_gate5_warrant": true,
|
|
"staged_file_count_excluding_isolation_receipt": 166,
|
|
"staged_file_count": 166,
|
|
"staged_files_tree_sha256": "6dfe597f7a5de1a7b7e5dfebe7e5bbc6b6392237bb48273254acfe3e1149da21",
|
|
"source_sampling_manifest_sha256": "a10146f86302f1d5c53493585278da89f17369ea9e5ba9091d6877dc8f84e52e",
|
|
"patch_sha256": "fa62ec6fecdf227fc1ac4c766bcdd16bb3d09f6a5e4d1a1c4c7a9d091442b76e"
|
|
},
|
|
"run_reviews_r2_receipt": {
|
|
"schema_version": 3,
|
|
"mode": "crosscheck",
|
|
"round": "r2",
|
|
"sandbox_mode": "macos_seatbelt_all_3_families_deny_repo_campaign_peer_read_and_workspace_write",
|
|
"access_policy_probe_verified": true,
|
|
"run_reviews_script_sha256": "e4af4a1ff9506628869ade9a15584bef30e160e1c0937c4c921c8993197159cb",
|
|
"stage_reviewer_workspace_script_sha256": "2b75c3e4a4c5814b4a6ceaac858fa0959bbe3ae30905a84886c4052c93ee7a42",
|
|
"workspace_isolation_receipt_sha256": "04a3e6e2357ca7c5e778133882e3809735d34c9ad66a8d7f0678ce3374d0a17f",
|
|
"staged_files_tree_sha256": "6dfe597f7a5de1a7b7e5dfebe7e5bbc6b6392237bb48273254acfe3e1149da21",
|
|
"patch_sha256": "fa62ec6fecdf227fc1ac4c766bcdd16bb3d09f6a5e4d1a1c4c7a9d091442b76e",
|
|
"brief_sha256": "cd7e749a83ea2673228c3e7f1d2cada3ffecc0fe36aaf4f3bef29409e604caca",
|
|
"rendered_brief_sha256": {
|
|
"codex": "99657b4d7275e2586d4bd81682c63cb46af982fdf84fd015785a6ea302b38ac2",
|
|
"gemini": "c38338a4ed46d19cad02c0259199df8a1dc1648106d0076866342d6948dc6e9d",
|
|
"claude": "99657b4d7275e2586d4bd81682c63cb46af982fdf84fd015785a6ea302b38ac2"
|
|
},
|
|
"models": {
|
|
"codex": "gpt-6-astra",
|
|
"gemini": "gemini-4-argon-high",
|
|
"claude": "claude-opus-5-5"
|
|
},
|
|
"cli_invocations": {
|
|
"codex": "sandbox-exec -f <codex_sb> codex exec --ephemeral --dangerously-bypass-approvals-and-sandbox -c mcp_servers={} --skip-git-repo-check -C <tmp_ws> (isolated CODEX_HOME)",
|
|
"gemini": "sandbox-exec -f <gemini_sb> agy -p ... --mode plan --disable-slash-commands --add-dir <tmp_ws>",
|
|
"claude": "sandbox-exec -f <claude_sb> claude -p ... --tools Read,Grep,Glob --setting-sources \"\" --output-format json --strict-mcp-config --mcp-config '{\"mcpServers\":{}}' --exclude-dynamic-system-prompt-sections --no-session-persistence --disable-slash-commands --add-dir <tmp_ws>"
|
|
},
|
|
"isolation_boundary_note": "External /tmp workspace staged outside git repo containing dev-only artifacts; all 3 reviewer CLIs (codex, agy, claude) run under separate macOS Seatbelt (/usr/bin/sandbox-exec) profiles denying file-read* and file-write* on the entire GitHub parent directory (covering the worktree, parent git repository, all sibling MLSysBook* checkouts, AIConfigs, and all other local repositories), campaign directory, /tmp/tax-cand* bundles, MLSysBook .claude directory, ~/.claude/{CLAUDE.md,rules,protected-checkouts.txt,projects}, ~/.codex, and peer reviewer /tmp I/O directories, denying file-write* on the staged workspace, and denying process-exec* on network CLI binaries (curl, gh, wget, scp, ssh, nc). Parent CLI processes retain outbound HTTPS solely to contact hosted LLM API endpoints under tool-level web/shell disablement. Pre-flight Seatbelt probes verify deny/allow enforcement before launch, and post-run UTF-8 scanning checks 561 distinct non-dev tokens (0 non-dev manifest tokens, 560 non-dev agreement item IDs, 1 held-out agreement suite ID, and 0 held_out:true checklist tokens).",
|
|
"non_dev_leak_scan_token_count": 561,
|
|
"non_dev_leak_scan_tokens_by_category": {
|
|
"non_dev_manifest_tokens": 0,
|
|
"non_dev_agreement_suite_ids": 1,
|
|
"non_dev_agreement_item_ids": 560,
|
|
"held_out_checklist_tokens": 0
|
|
},
|
|
"utf8_decode_failures": [],
|
|
"exit_codes": {
|
|
"codex": 0,
|
|
"gemini": 0,
|
|
"claude": 0
|
|
},
|
|
"output_sha256": {
|
|
"codex": "3a1c346b06233e264bdce6633ab5ae8a2277e6df0766115f20674f9a8a07940f",
|
|
"gemini": "a8b3057655bf02e087b6083ef3d52f72a5e11c7c7360e6a640be0447bcc5713a",
|
|
"claude": "9dca9628d44b1b4a170e6f7b2f3cf6860c59a941edc625f4700b6b91fe10c1b0"
|
|
},
|
|
"post_run_non_dev_leak_hits": 0,
|
|
"post_run_leaked_tokens": [],
|
|
"overall_rc": 0
|
|
},
|
|
"bucket_a_evidence_bundle": {
|
|
"cited_dev_unit_records": [
|
|
{
|
|
"unit_id": "P4-S2-CMU11868-02",
|
|
"area": "P4",
|
|
"stratum": "S2",
|
|
"provenance_family_id": "pf_cmu_llmsys",
|
|
"raw_phase3_cluster_id": "pf_cmu_llmsys",
|
|
"split": "dev",
|
|
"historical_phase3_unit": true,
|
|
"historical_phase6_split": "dev",
|
|
"origin": "external",
|
|
"editor_affiliated": false,
|
|
"source_type": "prose",
|
|
"passage_kind": "curated_summary",
|
|
"source_title": "CMU 11-868 Large Language Model Systems (Spring 2025)",
|
|
"locator": "Lectures 15-17: Structured Generation, RAG Serving, and Compound AI",
|
|
"passage": "Production LLM endpoints enforce output syntax via constrained decoding (compiling JSON schemas or context-free grammars into token-masking finite-state automata), orchestrate retrieval-augmented generation (vector indexing, approximate nearest-neighbor search, reranking, prompt assembly), and route queries across model cascades based on difficulty and cost.",
|
|
"authors": [
|
|
"Tianqi Chen",
|
|
"Lei Li"
|
|
]
|
|
},
|
|
{
|
|
"unit_id": "P4-S2-MIT65940-01",
|
|
"area": "P4",
|
|
"stratum": "S2",
|
|
"provenance_family_id": "pf_mit_han_tinyml",
|
|
"raw_phase3_cluster_id": "pf_mit_han_tinyml",
|
|
"split": "dev",
|
|
"historical_phase3_unit": true,
|
|
"historical_phase6_split": "dev",
|
|
"origin": "external",
|
|
"editor_affiliated": false,
|
|
"source_type": "prose",
|
|
"passage_kind": "curated_summary",
|
|
"source_title": "MIT 6.5940 TinyML and Efficient Deep Learning Computing (Fall 2024)",
|
|
"locator": "Lectures 4-8 & 14: Post-Training Quantization, Weight-Only INT4, and On-Device LLM Serving",
|
|
"passage": "Deploying models on edge SoCs, mobile NPUs, and microcontrollers requires post-training quantization (calibration, per-channel scaling, activation-aware weight quantization), static arena memory planning to avoid heap fragmentation under kilobyte SRAM ceilings, operator delegate fallback partitioning, and KV-cache eviction or streaming sinks for long contexts on device.",
|
|
"authors": [
|
|
"Song Han"
|
|
]
|
|
},
|
|
{
|
|
"unit_id": "P4-S3-SGLANG-2024",
|
|
"area": "P4",
|
|
"stratum": "S3",
|
|
"provenance_family_id": "pf_sglang_neurips24",
|
|
"raw_phase3_cluster_id": "pf_sglang_neurips24",
|
|
"split": "dev",
|
|
"historical_phase3_unit": true,
|
|
"historical_phase6_split": "dev",
|
|
"origin": "external",
|
|
"editor_affiliated": false,
|
|
"source_type": "prose",
|
|
"passage_kind": "curated_summary",
|
|
"source_title": "Zheng et al., SGLang: Efficient Execution of Structured Language Model Programs (NeurIPS 2024)",
|
|
"locator": "Sections 3-4: RadixAttention Prefix Caching and Compressed Finite-State Jump-Forward Decoding",
|
|
"passage": "Multi-turn agentic workflows, few-shot prompts, and tree-of-thought programs repeatedly share prompt prefixes. RadixAttention retains KV-cache blocks in an LRU radix tree across requests for automatic prefix reuse, while compressed finite-state machines fast-forward deterministic multi-token spans during grammar-constrained decoding.",
|
|
"authors": [
|
|
"Lianmin Zheng",
|
|
"Liangsheng Yin",
|
|
"Zhiqiang Xie",
|
|
"Chuyue Sun",
|
|
"Jeff Huang",
|
|
"Cody Hao Yu",
|
|
"Shiyi Cao",
|
|
"Christos Kozyrakis",
|
|
"Ion Stoica",
|
|
"Joseph E. Gonzalez",
|
|
"Clark Barrett",
|
|
"Ying Sheng"
|
|
]
|
|
},
|
|
{
|
|
"unit_id": "Q4-S2-STANFORD-CS329T-01",
|
|
"area": "Q4",
|
|
"stratum": "S2",
|
|
"provenance_family_id": "pf_stanford_cs329t",
|
|
"raw_phase3_cluster_id": "pf_stanford_cs329t",
|
|
"split": "dev",
|
|
"historical_phase3_unit": true,
|
|
"historical_phase6_split": "dev",
|
|
"origin": "external",
|
|
"editor_affiliated": false,
|
|
"source_type": "prose",
|
|
"passage_kind": "curated_summary",
|
|
"source_title": "Stanford CS329T Trustworthy Machine Learning",
|
|
"locator": "Modules on Certified Robustness, Input/Output Guardrails, and Red-Teaming",
|
|
"passage": "Assuring deployed ML systems combines pre-deployment adversarial stress testing and automated red-teaming with formal verification (SMT/MILP/bound propagation and randomized smoothing LP radii) and runtime input/output safety filters, content classifiers, and selective abstention when uncertainty or out-of-distribution indicators trip.",
|
|
"authors": [
|
|
"Sanmi Koyejo",
|
|
"Tatsu Hashimoto"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|