{"list":"other","total":51,"rows":[{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"1905.07830","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","created":"2019-05-19","cited_by_count":4100},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1705.03551","title":"TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension","created":"2017-05-09","cited_by_count":2955},{"id":"1911.11641","title":"PIQA: Reasoning about Physical Commonsense in Natural Language","created":"2019-11-26","cited_by_count":2883},{"id":"2211.05100","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","created":"2022-11-09","cited_by_count":1984},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"1706.02677","title":"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour","created":"2017-06-08","cited_by_count":1492},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897},{"id":"2401.06066","title":"DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models","created":"2024-01-11","cited_by_count":867},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399},{"id":"2401.15947","title":"MoE-LLaVA: Mixture of Experts for Large Vision-Language Models","created":"2024-01-29","cited_by_count":300},{"id":"1802.05799","title":"Horovod: fast and easy distributed deep learning in TensorFlow","created":"2018-02-15","cited_by_count":279},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"2407.06204","title":"A Survey on Mixture of Experts in Large Language Models","created":"2024-06-26","cited_by_count":237},{"id":"2209.05433","title":"FP8 Formats for Deep Learning","created":"2022-09-12","cited_by_count":196},{"id":"2411.12372","title":"RedPajama: an Open Dataset for Training Large Language Models","created":"2024-11-19","cited_by_count":182},{"id":"2211.10586","title":"Scaling Up Dataset Distillation to ImageNet-1K with Constant Memory","created":"2022-11-19","cited_by_count":181},{"id":"2402.07871","title":"Scaling Laws for Fine-Grained Mixture of Experts","created":"2024-02-12","cited_by_count":173},{"id":"2402.15627","title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs","created":"2024-02-23","cited_by_count":112},{"id":"2405.21015","title":"The rising costs of training frontier AI models","created":"2024-05-31","cited_by_count":95},{"id":"2203.12533","title":"Pathways: Asynchronous Distributed Dataflow for ML","created":"2022-03-23","cited_by_count":82},{"id":"1901.05758","title":"Analysis of Large-Scale Multi-Tenant GPU Clusters for DNN Training Workloads","created":"2019-01-17","cited_by_count":79},{"id":"2407.04153","title":"Mixture of A Million Experts","created":"2024-07-04","cited_by_count":72},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68},{"id":"2304.03946","title":"FlexMoE: Scaling Large-scale Sparse Pre-trained Model Training via Dynamic Device Placement","created":"2023-04-08","cited_by_count":53},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49},{"id":"2410.07524","title":"Upcycling Large Language Models into Mixture of Experts","created":"2024-10-10","cited_by_count":49},{"id":"2403.07648","title":"Characterization of Large Language Model Development in the Datacenter","created":"2024-03-12","cited_by_count":46},{"id":"2309.08125","title":"Oobleck: Resilient Distributed Training of Large Models Using Pipeline Templates","created":"2023-09-15","cited_by_count":40},{"id":"2003.09518","title":"Deep Learning Training in Facebook Data Centers: Design of Scale-up and Scale-out Systems","created":"2020-03-20","cited_by_count":38},{"id":"2408.10681","title":"HMoE: Heterogeneous Mixture of Experts for Language Modeling","created":"2024-08-20","cited_by_count":29},{"id":"2410.21680","title":"Revisiting Reliability in Large-Scale Machine Learning Research Clusters","created":"2024-10-29","cited_by_count":28},{"id":"2407.20143","title":"ByteCheckpoint: A Unified Checkpointing System for Large Foundation Model Development","created":"2024-07-29","cited_by_count":24},{"id":"1709.10432","title":"Convergence Analysis of Distributed Stochastic Gradient Descent with Shuffling","created":"2017-09-29","cited_by_count":23},{"id":"1905.12334","title":"Mixed Precision Training With 8-bit Floating Point","created":"2019-05-29","cited_by_count":23},{"id":"2401.00134","title":"Unicron: Economizing Self-Healing LLM Training at Scale","created":"2023-12-30","cited_by_count":16},{"id":"2406.10707","title":"DataStates-LLM: Lazy Asynchronous Checkpointing for Large Language Models","created":"2024-06-15","cited_by_count":16},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14},{"id":"2011.02999","title":"CPR: Understanding and Improving Failure Tolerant Training for Deep Learning Recommendation with Partial Recovery","created":"2020-11-05","cited_by_count":11},{"id":"2405.03637","title":"Collage: Light-Weight Low-Precision Strategy for LLM Training","created":"2024-05-06","cited_by_count":10},{"id":"2405.18710","title":"To FP8 and Back Again: Quantifying Reduced Precision Effects on LLM Training Stability","created":"2024-05-29","cited_by_count":9},{"id":"1810.07354","title":"Fault Tolerance in Iterative-Convergent Machine Learning","created":"2018-10-17","cited_by_count":7},{"id":"2406.13768","title":"FastPersist: Accelerating Model Checkpointing in Deep Learning","created":"2024-06-19","cited_by_count":6},{"id":"2306.11800","title":"Inshrinkerator: Compressing Deep Learning Training Checkpoints via Dynamic Quantization","created":"2023-06-20","cited_by_count":2}],"next_cursor":null}