mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-14 18:24:37 -06:00
561 lines
12 KiB
YAML
561 lines
12 KiB
YAML
schema_version: 1
|
|
study_id: taxonomy-method-two-area-test-2026-11
|
|
preregistered_date: '2026-11-02'
|
|
random_seed: 20261103
|
|
test_areas:
|
|
- code: P4
|
|
id: serving
|
|
label: Deployment & serving
|
|
subject_count: 9
|
|
subjects:
|
|
- P4.1 (serving-architecture)
|
|
- P4.2 (compound-ai)
|
|
- P4.3 (latency-goodput)
|
|
- P4.4 (request-scheduling)
|
|
- P4.5 (llm-inference)
|
|
- P4.6 (structured-generation)
|
|
- P4.7 (test-time-compute)
|
|
- P4.8 (retrieval-rag)
|
|
- P4.9 (on-device-deployment)
|
|
published_0_4_draft_concept_count: 95
|
|
unit_counts_by_split:
|
|
dev: 142
|
|
held_out_test: 116
|
|
reserve: 47
|
|
unit_counts_by_stratum:
|
|
S1: 56
|
|
S2: 36
|
|
S3: 173
|
|
S4: 38
|
|
S5: 2
|
|
- code: Q4
|
|
id: safety-assurance
|
|
label: Safety & assurance
|
|
subject_count: 4
|
|
subjects:
|
|
- Q4.1 (hazards-operating-domain)
|
|
- Q4.2 (runtime-enforcement)
|
|
- Q4.3 (verification-validation)
|
|
- Q4.4 (safety-cases)
|
|
published_0_4_draft_concept_count: 30
|
|
unit_counts_by_split:
|
|
dev: 58
|
|
held_out_test: 10
|
|
reserve: 9
|
|
unit_counts_by_stratum:
|
|
S1: 55
|
|
S2: 8
|
|
S3: 6
|
|
S4: 5
|
|
S5: 3
|
|
artifact_hashes:
|
|
dev_sources_sha256: 0849d619afb0e745fdde9f5fef715d67cd3d49da31459e2a0b502dafab866df0
|
|
held_out_slice_sha256: 221fb94438db2d5e0f78f4dbb73c99c081e73d8788c89cbd12468f6a054607c7
|
|
total_units: 382
|
|
dev_units: 200
|
|
held_out_test_units: 126
|
|
reserve_units: 56
|
|
enumerated_clusters:
|
|
- provenance_family_id: pf_amodei_concrete_safety
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_barroso_wsc
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_berkeley_aisafety
|
|
area: Q4
|
|
stratum: S2
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cmu_llmsys
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_cov_courses_2026_AIE
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 4
|
|
- provenance_family_id: pf_cov_courses_2026_B294
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_BAG
|
|
area: Q4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_C442
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 4
|
|
- provenance_family_id: pf_cov_courses_2026_C779
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_C868
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 7
|
|
- provenance_family_id: pf_cov_courses_2026_DLS
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_DMLS
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_HAR
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_K17
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_cov_courses_2026_M594
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_cov_courses_2026_MLDP
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_MLS26
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_NIST
|
|
area: Q4
|
|
stratum: S2
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_OSDI
|
|
area: P4
|
|
stratum: S2
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_RAT
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_S329
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_cov_courses_2026_S336
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_SEI
|
|
area: Q4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_USC
|
|
area: Q4
|
|
stratum: S2
|
|
split: reserve
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_courses_2026_UT378
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_cov_mlsys_25_26_mlsys2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 16
|
|
- provenance_family_id: pf_cov_mlsys_25_26_mlsys2026
|
|
area: P4
|
|
stratum: S3
|
|
split: reserve
|
|
unit_count: 45
|
|
- provenance_family_id: pf_cov_sysvenues_2025_asplos2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 13
|
|
- provenance_family_id: pf_cov_sysvenues_2025_eurosys2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 8
|
|
- provenance_family_id: pf_cov_sysvenues_2025_nsdi2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_cov_sysvenues_2025_osdi2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_cov_sysvenues_2025_sosp2025
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 9
|
|
- provenance_family_id: pf_cov_sysvenues_2026_heldout_asplos2026
|
|
area: P4
|
|
stratum: S3
|
|
split: held_out_test
|
|
unit_count: 25
|
|
- provenance_family_id: pf_cov_sysvenues_2026_heldout_eurosys2026
|
|
area: P4
|
|
stratum: S3
|
|
split: held_out_test
|
|
unit_count: 20
|
|
- provenance_family_id: pf_cov_sysvenues_2026_heldout_nsdi2026
|
|
area: P4
|
|
stratum: S3
|
|
split: held_out_test
|
|
unit_count: 12
|
|
- provenance_family_id: pf_cov_sysvenues_2026_heldout_osdi2026
|
|
area: P4
|
|
stratum: S3
|
|
split: held_out_test
|
|
unit_count: 16
|
|
- provenance_family_id: pf_distserve_osdi24
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_frontier_scaling_policies
|
|
area: Q4
|
|
stratum: S4
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_huyen_dmls
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_iso42001_pas8800
|
|
area: Q4
|
|
stratum: S5
|
|
split: reserve
|
|
unit_count: 1
|
|
- provenance_family_id: pf_iso_automotive_ul4600
|
|
area: Q4
|
|
stratum: S5
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_k8s_ray_serving
|
|
area: P4
|
|
stratum: S4
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_kalra_statistical_safety
|
|
area: Q4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_katz_verification
|
|
area: Q4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_leveson_stpa
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mit_han_tinyml
|
|
area: P4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlperf_inference
|
|
area: P4
|
|
stratum: S5
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol1/02_system_paradigms
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol1/05_nn_computation
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol1/06_nn_architectures
|
|
area: P4
|
|
stratum: S1
|
|
split: reserve
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol1/12_benchmarking
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol1/13_model_serving
|
|
area: P4
|
|
stratum: S1
|
|
split: held_out_test
|
|
unit_count: 5
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol2/08_fleet_orchestration
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol2/10_inference
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 7
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol2/11_edge_intelligence
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol2/14_robust_ai
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v1_v2_vol2/16_responsible_ai
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/02_foundation_model
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 6
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/03_test_time_compute
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 7
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/04_context_engineering
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/05_kv_cache
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 7
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/06_long_term_memory
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 4
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/10_agent_harness
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/15_trajectory_curation
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/17_rlvr
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol3/19_agent_economics
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/01_boundary
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 4
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/02_body
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/03_brain
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/04_nervous
|
|
area: Q4
|
|
stratum: S1
|
|
split: held_out_test
|
|
unit_count: 2
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/05_data
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/06_training
|
|
area: P4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 2
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/07_evaluation
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 5
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/08_perception
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/10_intent
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/11_planning
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/12_enforcement
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 7
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/13_placement
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/14_intervention
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/15_verification
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 6
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/16_release
|
|
area: Q4
|
|
stratum: S1
|
|
split: reserve
|
|
unit_count: 7
|
|
- provenance_family_id: pf_mlsysbook_v3_v4_vol4/18_frontier
|
|
area: Q4
|
|
stratum: S1
|
|
split: dev
|
|
unit_count: 3
|
|
- provenance_family_id: pf_nist_ai_rmf
|
|
area: Q4
|
|
stratum: S5
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_onnx_runtime_docs
|
|
area: P4
|
|
stratum: S4
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_orca_osdi22
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_sarathi_osdi24
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_sglang_neurips24
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_simplex_cbf_control
|
|
area: Q4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_slora_mlsys24
|
|
area: P4
|
|
stratum: S3
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_specdec_icml23
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_staffml_heldout_p4
|
|
area: P4
|
|
stratum: S4
|
|
split: held_out_test
|
|
unit_count: 36
|
|
- provenance_family_id: pf_staffml_heldout_q4
|
|
area: Q4
|
|
stratum: S4
|
|
split: held_out_test
|
|
unit_count: 2
|
|
- provenance_family_id: pf_staffml_heldout_q4_ext
|
|
area: Q4
|
|
stratum: S4
|
|
split: held_out_test
|
|
unit_count: 2
|
|
- provenance_family_id: pf_stanford_cs329t
|
|
area: Q4
|
|
stratum: S2
|
|
split: dev
|
|
unit_count: 1
|
|
- provenance_family_id: pf_stanford_cs336
|
|
area: P4
|
|
stratum: S2
|
|
split: held_out_test
|
|
unit_count: 1
|
|
- provenance_family_id: pf_ut_cs378
|
|
area: P4
|
|
stratum: S2
|
|
split: reserve
|
|
unit_count: 1
|
|
- provenance_family_id: pf_vllm_sosp23
|
|
area: P4
|
|
stratum: S3
|
|
split: dev
|
|
unit_count: 1
|
|
evaluation_metrics_and_thresholds:
|
|
held_out_topic_coverage_pct_min: 85.0
|
|
seeded_gap_detection_pct_min: 85.0
|
|
subject_kappa_min: 0.7
|
|
hooper_consistency_pct_min: 40.0
|
|
rolling_consistency_pct_min: 57.0
|
|
alpha_masi_min: 0.4
|
|
max_single_concept_share_pct_max: 15.0
|
|
specific_concept_tag_ratio_pct_min: 75.0
|
|
vocabulary_size_ratio_bounds:
|
|
- 0.7
|
|
- 1.35
|
|
failure_criterion: The synthesized method does no better than the current method if, on the pre-registered
|
|
held-out slice (held_out_test), the rebuilt P4 and Q4 vocabulary fails to improve or match held-out
|
|
topic coverage (covered_pct) and multi-rater indexing consistency (hooper_consistency_pct, rolling_consistency_pct,
|
|
alpha_masi, subject_kappa) without violating the anti-gaming specificity cap (max_single_concept_share_pct
|
|
<= 15.0%) or exceeding 1.35x the published concept count.
|