cs249r_book/mlsysim/docs/references.bib

@inproceedings{abadi2016,
  title = {Deep Learning With Differential Privacy},
  author = {Abadi, Martin and Chu, Andy and Goodfellow, Ian and others},
  year = {2016},
  booktitle = {Proceedings of the 2016 ACM SIGSAC Conference on Computer and Communications Security},
  publisher = {ACM},
  pages = {308--318},
  doi = {10.1145/2976749.2978318},
  url = {https://doi.org/10.1145/2976749.2978318},
  source = {Crossref},
}

@inproceedings{agrawal2025,
  title = {Taming {Throughput-Latency} Tradeoff in {LLM} Inference with {Sarathi-Serve}},
  author = {
    Agrawal, Amey and Kedia, Nitin and Panwar, Ashish and Mohan, Jayashree and Kwatra, Nipun and
    Gulavani, Bhargav and Tumanov, Alexey and Ramjee, Ramachandran
  },
  year = {2024},
  month = jul,
  booktitle = {18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)},
  publisher = {USENIX Association},
  pages = {117--134},
  isbn = {978-1-939133-40-3},
  url = {https://www.usenix.org/conference/osdi24/presentation/agrawal},
}

@misc{amodei2018ai,
  title = {{AI} and Compute},
  author = {Amodei, Dario and Hernandez, Danny},
  year = {2018},
  url = {https://openai.com/research/ai-and-compute},
  howpublished = {OpenAI Blog},
}

@book{barroso2019,
  title = {The Datacenter as a Computer},
  author = {Barroso, Luiz Andr{\'e} and H{\"o}lzle, Urs and Ranganathan, Parthasarathy},
  year = {2018},
  publisher = {Springer International Publishing},
  series = {Synthesis Lectures on Computer Architecture},
  doi = {10.1007/978-3-031-01761-2},
  isbn = {9783031006333, 9783031017612},
  issn = {1935-3235, 1935-3243},
  url = {https://doi.org/10.1007/978-3-031-01761-2},
  subtitle = {Designing Warehouse-Scale Machines},
  source = {Crossref},
  edition = {3rd},
}

@article{chowdhery2022palm,
  title = {{PaLM}: Scaling Language Modeling with Pathways},
  author = {Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and others},
  year = {2023},
  journal = {Journal of Machine Learning Research},
  volume = {24},
  number = {240},
  pages = {1--113},
}

@article{daly2006,
  title = {A Higher Order Estimate of the Optimum Checkpoint Interval for Restart Dumps},
  author = {Daly, John T.},
  year = {2006},
  journal = {Future Gener. Comput. Syst.},
  publisher = {Elsevier BV},
  volume = {22},
  number = {3},
  pages = {303--312},
  doi = {10.1016/j.future.2004.11.016},
  issn = {0167-739X},
  url = {https://doi.org/10.1016/j.future.2004.11.016},
  source = {Crossref},
}

@inproceedings{dao2022,
  title = {FlashAttention: Fast and Memory-Efficient Exact Attention With IO-Awareness},
  author = {Dao, Tri and Fu, Daniel Y. and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
  year = {2022},
  booktitle = {Advances in Neural Information Processing Systems 35},
  publisher = {Neural Information Processing Systems Foundation, Inc. (NeurIPS)},
  pages = {16344--16359},
  doi = {10.52202/068431-1189},
  url = {https://doi.org/10.52202/068431-1189},
  source = {Crossref},
}

@article{dean2012large,
  title = {Large Scale Distributed Deep Networks},
  author = {Dean, Jeffrey and Corrado, Greg S. and Monga, Rajat and others},
  year = {2012},
  journal = {Advances in Neural Information Processing Systems},
  volume = {25},
}

@article{dean2013,
  title = {The Tail at Scale},
  author = {Dean, Jeffrey and Barroso, Luiz Andr{\'e}},
  year = {2013},
  journal = {Communications of the ACM},
  publisher = {Association for Computing Machinery (ACM)},
  volume = {56},
  number = {2},
  pages = {74--80},
  doi = {10.1145/2408776.2408794},
  issn = {0001-0782, 1557-7317},
  url = {https://doi.org/10.1145/2408776.2408794},
  source = {Crossref},
}

@inproceedings{eisenman2022checknrun,
  title = {Check-N-Run: a Checkpointing System for Training Deep Learning Recommendation Models},
  author = {
    Eisenman, Assaf and Matam, Kiran Kumar and Ingram, Steven and Mudigere, Dheevatsa and
    Krishnamoorthi, Raghuraman and Nair, Krishnakumar and Smelyanskiy, Misha and Annavaram, Murali
  },
  year = {2022},
  booktitle = {
    Proceedings of the 19th USENIX Symposium on Networked Systems Design and Implementation (NSDI)
  },
  publisher = {USENIX Association},
}

@article{fedus2022switch,
  title = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity},
  author = {Fedus, William and Zoph, Barret and Shazeer, Noam},
  year = {2022},
  journal = {Journal of Machine Learning Research},
  volume = {23},
  number = {120},
  pages = {1--39},
  url = {https://www.jmlr.org/papers/v23/21-0998.html},
}

@inbook{gholami2022,
  title = {A Survey of Quantization Methods for Efficient Neural Network Inference},
  author = {Gholami, Amir and Kim, Sehoon and Dong, Zhen and others},
  year = {2021},
  journal = {arXiv preprint arXiv:2103.13630},
  booktitle = {Low-Power Computer Vision},
  publisher = {Chapman and Hall/CRC},
  pages = {291--326},
  doi = {10.1201/9781003162810-13},
  isbn = {9781003162810},
  url = {https://doi.org/10.1201/9781003162810-13},
  source = {Crossref},
}

@article{han2015deep,
  title = {
    Deep Compression: Compressing Deep Neural Networks with Pruning, Trained Quantization and
    Huffman Coding
  },
  author = {Han, Song and Mao, Huizi and Dally, William J.},
  year = {2015},
  journal = {arXiv preprint arXiv:1510.00149},
}

@book{hennessy2019architecture,
  title = {Computer Architecture: A Quantitative Approach},
  author = {Hennessy, John L. and Patterson, David A.},
  year = {2019},
  publisher = {Morgan Kaufmann},
  isbn = {978-0128119051},
  edition = {6th},
}

@inproceedings{hoffmann2022chinchilla,
  title = {Training Compute-Optimal Large Language Models},
  author = {Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and others},
  year = {2022},
  journal = {arXiv preprint arXiv:2203.15556},
  booktitle = {Advances in Neural Information Processing Systems 35},
  publisher = {Neural Information Processing Systems Foundation, Inc. (NeurIPS)},
  pages = {30016--30030},
  doi = {10.52202/068431-2176},
  url = {https://doi.org/10.52202/068431-2176},
  source = {Crossref},
}

@inproceedings{isaev2023,
  title = {
    Calculon: A Methodology and Tool for High-Level Co-Design of Systems and Large Language Models
  },
  author = {Isaev, Mikhail and McDonald, Nic and Dennison, Larry and Vuduc, Richard},
  year = {2023},
  booktitle = {
    Proceedings of the International Conference for High Performance Computing, Networking, Storage
    and Analysis
  },
  publisher = {ACM},
  pages = {1--14},
  doi = {10.1145/3581784.3607102},
  url = {https://doi.org/10.1145/3581784.3607102},
  source = {Crossref},
}

@inproceedings{jouppi2017,
  title = {In-Datacenter Performance Analysis of a Tensor Processing Unit},
  author = {Jouppi, Norman P. and Young, Cliff and Patil, Nishant and others},
  year = {2017},
  booktitle = {Proceedings of the 44th Annual International Symposium on Computer Architecture},
  publisher = {ACM},
  pages = {1--12},
  doi = {10.1145/3079856.3080246},
  url = {https://doi.org/10.1145/3079856.3080246},
  source = {Crossref},
}

@article{kaplan2020scaling,
  title = {Scaling Laws for Neural Language Models},
  author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and others},
  year = {2020},
  journal = {arXiv preprint arXiv:2001.08361},
}

@article{korthikanti2023,
  title = {Reducing Activation Recomputation in Large Transformer Models},
  author = {
    Korthikanti, Vijay Anand and Casper, Jared and Lym, Sangkug and McAfee, Lawrence and Andersch,
    Michael and Shoeybi, Mohammad and Catanzaro, Bryan
  },
  year = {2023},
  journal = {Proceedings of Machine Learning and Systems},
  volume = {5},
  url = {
    https://proceedings.mlsys.org/paper_files/paper/2023/hash/80083951326cf5b35e5100260d64ed81-Abstract-mlsys2023.html
  },
}

@inproceedings{kwon2023,
  title = {Efficient Memory Management for Large Language Model Serving With PagedAttention},
  author = {Kwon, Woosuk and Li, Zhuohan and Zhuang, Siyuan and others},
  year = {2023},
  booktitle = {Proceedings of the 29th Symposium on Operating Systems Principles},
  publisher = {ACM},
  pages = {611--626},
  doi = {10.1145/3600006.3613165},
  url = {https://doi.org/10.1145/3600006.3613165},
  source = {Crossref},
}

@article{leiserson1985,
  title = {Fat-Trees: Universal Networks for Hardware-Efficient Supercomputing},
  author = {Leiserson, Charles E.},
  year = {1985},
  journal = {IEEE Transactions on Computers},
  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},
  volume = {C-34},
  number = {10},
  pages = {892--901},
  doi = {10.1109/tc.1985.6312192},
  issn = {0018-9340},
  url = {https://doi.org/10.1109/tc.1985.6312192},
  source = {Crossref},
}

@article{lepikhin2020gshard,
  title = {{GShard}: Scaling Giant Models with Conditional Computation and Automatic Sharding},
  author = {
    Lepikhin, Dmitry and Lee, HyoukJoong and Xu, Yuanzhong and Chen, Dehao and Firat, Orhan and
    Huang, Yanping and Krikun, Maxim and Shazeer, Noam and Chen, Zhifeng
  },
  year = {2020},
  journal = {arXiv preprint arXiv:2006.16668},
  url = {https://arxiv.org/abs/2006.16668},
}

@inproceedings{leviathan2023fast,
  title = {Fast Inference from Transformers via Speculative Decoding},
  author = {Leviathan, Yaniv and Kalman, Matan and Matias, Yossi},
  year = {2023},
  booktitle = {Proceedings of the 40th International Conference on Machine Learning (ICML)},
  publisher = {PMLR},
  url = {https://arxiv.org/abs/2211.17192},
}

@article{little1961,
  title = {A Proof for the Queuing Formula: <I>L</i> = \ensuremath{\Lambda}<I>W</i>},
  author = {Little, John D. C.},
  year = {1961},
  journal = {Oper. Res.},
  publisher = {Institute for Operations Research and the Management Sciences (INFORMS)},
  volume = {9},
  number = {3},
  pages = {383--387},
  doi = {10.1287/opre.9.3.383},
  issn = {0030-364X, 1526-5463},
  url = {https://doi.org/10.1287/opre.9.3.383},
  source = {Crossref},
}

@article{mattson2020,
  title = {MLPerf: An Industry Standard Benchmark Suite for Machine Learning Performance},
  author = {Mattson, Peter and Cheng, Christine and Diamos, Gregory and others},
  year = {2020},
  journal = {IEEE Micro},
  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},
  volume = {40},
  number = {2},
  pages = {8--16},
  doi = {10.1109/mm.2020.2974843},
  issn = {0272-1732, 1937-4143},
  url = {https://doi.org/10.1109/mm.2020.2974843},
  source = {Crossref},
}

@book{mlsysbook2025,
  title = {
    Machine Learning Systems: Principles and Practices of Engineering Artificially Intelligent
    Systems
  },
  author = {Reddi, Vijay Janapa and others},
  year = {2025},
  publisher = {Harvard University},
  url = {https://mlsysbook.ai},
}

@article{mohan2021,
  title = {Analyzing and Mitigating Data Stalls in DNN Training},
  author = {Mohan, Jayashree and Phanishayee, Amar and Raniwala, Ashish and Chidambaram, Vijay},
  year = {2021},
  journal = {Proc. VLDB Endow.},
  publisher = {Association for Computing Machinery (ACM)},
  volume = {14},
  number = {5},
  pages = {771--784},
  doi = {10.14778/3446095.3446100},
  issn = {2150-8097},
  url = {https://doi.org/10.14778/3446095.3446100},
  source = {Crossref},
}

@article{murray2021tf,
  title = {{tf.data}: A Machine Learning Data Processing Framework},
  author = {Murray, Derek G. and Simsa, Jiri and Klimovic, Ana and Indyk, Ihor},
  year = {2021},
  journal = {Proceedings of the VLDB Endowment},
  publisher = {VLDB Endowment},
  volume = {14},
  number = {12},
}

@inproceedings{narayanan2021,
  title = {Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM},
  author = {Narayanan, Deepak and Shoeybi, Mohammad and Casper, Jared and others},
  year = {2021},
  booktitle = {
    Proceedings of the International Conference for High Performance Computing, Networking, Storage
    and Analysis
  },
  publisher = {ACM},
  pages = {1--15},
  doi = {10.1145/3458817.3476209},
  url = {https://doi.org/10.1145/3458817.3476209},
  source = {Crossref},
}

@misc{nvidia2023h100,
  title = {{NVIDIA H100 Tensor Core GPU} Datasheet},
  author = {{NVIDIA Corporation}},
  year = {2023},
  note = {Accessed: 2024-06-15},
  howpublished = {\url{https://www.nvidia.com/en-us/data-center/h100/}},
}

@inproceedings{parashar2019,
  title = {Timeloop: A Systematic Approach to DNN Accelerator Evaluation},
  author = {
    Parashar, Angshuman and Raina, Priyanka and Shao, Yakun Sophia and Chen, Yu-Hsin and Emer, Joel
    and others
  },
  year = {2019},
  booktitle = {2019 IEEE International Symposium on Performance Analysis of Systems and Software (ISPASS)},
  publisher = {IEEE},
  pages = {304--315},
  doi = {10.1109/ispass.2019.00042},
  url = {https://doi.org/10.1109/ispass.2019.00042},
  source = {Crossref},
}

@article{patarasuk2009,
  title = {Bandwidth optimal all-reduce algorithms for clusters of workstations},
  author = {Patarasuk, Pitch and Yuan, Xin},
  year = {2009},
  journal = {Journal of Parallel and Distributed Computing},
  publisher = {Elsevier},
  volume = {69},
  number = {2},
  pages = {117--124},
  doi = {10.1016/j.jpdc.2008.09.002},
  url = {https://doi.org/10.1016/j.jpdc.2008.09.002},
}

@inproceedings{patel2024,
  title = {Splitwise: Efficient Generative LLM Inference Using Phase Splitting},
  author = {
    Patel, Pratyush and Choukse, Esha and Zhang, Chaojie and Shah, Aashaka and Goiri, {\'I}{\~n}igo
    and Maleki, Saeed and Bianchini, Ricardo
  },
  year = {2024},
  booktitle = {2024 ACM/IEEE 51st Annual International Symposium on Computer Architecture (ISCA)},
  publisher = {IEEE},
  pages = {118--132},
  doi = {10.1109/isca59077.2024.00019},
  url = {https://doi.org/10.1109/isca59077.2024.00019},
  source = {Crossref},
}

@article{patterson2021carbon,
  title = {Carbon Emissions and Large Neural Network Training},
  author = {Patterson, David and Gonzalez, Joseph and Le, Quoc and others},
  year = {2021},
  journal = {arXiv preprint arXiv:2104.10350},
}

@inproceedings{pope2023llm,
  title = {Efficiently Scaling Transformer Inference},
  author = {Pope, Reiner and Douglas, Sholto and Chowdhery, Aakanksha and others},
  year = {2023},
  booktitle = {Proceedings of Machine Learning and Systems (MLSys)},
  publisher = {mlsys.org},
  volume = {5},
  url = {
    https://proceedings.mlsys.org/paper_files/paper/2023/hash/c4be71ab8d24cdfb45e3d06dbfca2780-Abstract-mlsys2023.html
  },
}

@inproceedings{rajbhandari2020,
  title = {ZeRO: Memory Optimizations Toward Training Trillion Parameter Models},
  author = {Rajbhandari, Samyam and Rasley, Jeff and Ruwase, Olatunji and He, Yuxiong},
  year = {2020},
  booktitle = {
    SC20: International Conference for High Performance Computing, Networking, Storage and Analysis
  },
  publisher = {IEEE},
  pages = {1--16},
  doi = {10.1109/sc41405.2020.00024},
  url = {https://doi.org/10.1109/sc41405.2020.00024},
  source = {Crossref},
}

@inproceedings{rasley2020,
  title = {DeepSpeed},
  author = {Rasley, Jeff and Rajbhandari, Samyam and Ruwase, Olatunji and He, Yuxiong},
  year = {2020},
  booktitle = {
    Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery \&amp; Data
    Mining
  },
  publisher = {ACM},
  pages = {3505--3506},
  doi = {10.1145/3394486.3406703},
  url = {https://doi.org/10.1145/3394486.3406703},
  subtitle = {System Optimizations Enable Training Deep Learning Models With Over 100 Billion Parameters},
  source = {Crossref},
}

@article{shazeer2017outrageously,
  title = {Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},
  author = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and others},
  year = {2017},
  journal = {arXiv preprint arXiv:1701.06538},
}

@article{shoeybi2019megatron,
  title = {{Megatron-LM}: Training Multi-Billion Parameter Language Models Using Model Parallelism},
  author = {
    Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared
    and Catanzaro, Bryan
  },
  year = {2019},
  journal = {arXiv preprint arXiv:1909.08053},
}

@inproceedings{snell2025scaling,
  title = {Scaling {LLM} Test-Time Compute Optimally can be More Effective than Scaling Model Parameters},
  author = {Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral},
  year = {2025},
  booktitle = {Proceedings of the 13th International Conference on Learning Representations (ICLR)},
  publisher = {OpenReview.net},
  note = {Oral presentation. arXiv:2408.03314},
}

@article{williams2009,
  title = {Roofline},
  author = {Williams, Samuel and Waterman, Andrew and Patterson, David},
  year = {2009},
  journal = {Communications of the ACM},
  publisher = {Association for Computing Machinery (ACM)},
  volume = {52},
  number = {4},
  pages = {65--76},
  doi = {10.1145/1498765.1498785},
  issn = {0001-0782, 1557-7317},
  url = {https://doi.org/10.1145/1498765.1498785},
  subtitle = {An Insightful Visual Performance Model for Multicore Architectures},
  source = {Crossref},
}

@inproceedings{won2023,
  title = {
    ASTRA-Sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-Model Training
    at Scale
  },
  author = {
    Won, William and Heo, Taekyung and Rashidi, Saeed and Sridharan, Srinivas and Srinivasan,
    Sudarshan and Krishna, Tushar
  },
  year = {2023},
  booktitle = {2023 IEEE International Symposium on Performance Analysis of Systems and Software (ISPASS)},
  publisher = {IEEE},
  pages = {283--294},
  doi = {10.1109/ispass57527.2023.00035},
  url = {https://doi.org/10.1109/ispass57527.2023.00035},
  source = {Crossref},
}

@inproceedings{wu2019,
  title = {Accelergy: An Architecture-Level Energy Estimation Methodology for Accelerator Designs},
  author = {Wu, Yannan Nellie and Emer, Joel S. and Sze, Vivienne},
  year = {2019},
  booktitle = {2019 IEEE/ACM International Conference on Computer-Aided Design (ICCAD)},
  publisher = {IEEE},
  pages = {1--8},
  doi = {10.1109/iccad45719.2019.8942149},
  url = {https://doi.org/10.1109/iccad45719.2019.8942149},
  source = {Crossref},
}

@article{young1974,
  title = {A First Order Approximation to the Optimum Checkpoint Interval},
  author = {Young, John W.},
  year = {1974},
  journal = {Communications of the ACM},
  publisher = {Association for Computing Machinery (ACM)},
  volume = {17},
  number = {9},
  pages = {530--531},
  doi = {10.1145/361147.361115},
  issn = {0001-0782, 1557-7317},
  url = {https://doi.org/10.1145/361147.361115},
  source = {Crossref},
}

@inproceedings{zheng2024sglang,
  title = {SGLang: Efficient Execution of Structured Language Model Programs},
  author = {Zheng, Lianmin and Yin, Liangsheng and Xie, Zhiqiang and others},
  year = {2024},
  journal = {arXiv preprint arXiv:2312.07104},
  booktitle = {Advances in Neural Information Processing Systems 37},
  publisher = {Neural Information Processing Systems Foundation, Inc. (NeurIPS)},
  pages = {62557--62583},
  doi = {10.52202/079017-2000},
  url = {https://doi.org/10.52202/079017-2000},
  note = {Introduces RadixAttention for automatic prompt caching via prefix tree matching},
  source = {Crossref},
}

@inproceedings{zhong2024distserve,
  title = {
    {DistServe}: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model
    Serving
  },
  author = {
    Zhong, Yinmin and Liu, Shengyu and Chen, Junda and Hu, Jianbo and Zhu, Yibo and Liu, Xuanzhe
    and Jin, Xin and Zhang, Hao
  },
  year = {2024},
  month = jul,
  booktitle = {18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)},
  publisher = {USENIX Association},
  pages = {193--210},
  isbn = {978-1-939133-40-3},
  url = {https://www.usenix.org/conference/osdi24/presentation/zhong-yinmin},
}