{"total":169,"rows":[{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326,"area":null},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073,"area":"moe"},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631,"area":"moe"},{"id":"2208.07339","title":"LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale","created":"2022-08-15","cited_by_count":906,"area":null},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897,"area":"wm"},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2202.09368","title":"Mixture-of-Experts with Expert Choice Routing","created":"2022-02-18","cited_by_count":623,"area":"moe"},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535,"area":null},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399,"area":"moe"},{"id":"2501.13928","title":"Fast3R: Towards 3D Reconstruction of 1000+ Images in One Forward Pass","created":"2025-01-23","cited_by_count":367,"area":null},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2104.08378","title":"Accelerating Sparse Deep Neural Networks","created":"2021-04-16","cited_by_count":223,"area":null},{"id":"2203.17189","title":"Scaling Up Models and Data with $\\texttt{t5x}$ and $\\texttt{seqio}$","created":"2022-03-31","cited_by_count":187,"area":null},{"id":"2202.11176","title":"A New Generation of Perspective API: Efficient Multilingual Character-level Transformers","created":"2022-02-22","cited_by_count":185,"area":null},{"id":"2111.10050","title":"Combined Scaling for Zero-shot Transfer Learning","created":"2021-11-19","cited_by_count":182,"area":null},{"id":"2402.07871","title":"Scaling Laws for Fine-Grained Mixture of Experts","created":"2024-02-12","cited_by_count":173,"area":"moe"},{"id":"2111.10952","title":"ExT5: Towards Extreme Multi-Task Scaling for Transfer Learning","created":"2021-11-22","cited_by_count":161,"area":null},{"id":"2106.12672","title":"Charformer: Fast Character Transformers via Gradient-based Subword Tokenization","created":"2021-06-23","cited_by_count":160,"area":null},{"id":"2309.05444","title":"Pushing Mixture of Experts to the Limit: Extremely Parameter Efficient MoE for Instruction Tuning","created":"2023-09-11","cited_by_count":150,"area":"moe"},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2104.12369","title":"PanGu-$\\alpha$: Large-scale Autoregressive Pretrained Chinese Language Models with Auto-parallel Computation","created":"2021-04-26","cited_by_count":134,"area":null},{"id":"2112.03052","title":"Scaling Up Influence Functions","created":"2021-12-06","cited_by_count":132,"area":null},{"id":"2103.13262","title":"FastMoE: A Fast Mixture-of-Expert Training System","created":"2021-03-24","cited_by_count":112,"area":"moe"},{"id":"2105.04663","title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","created":"2021-05-10","cited_by_count":111,"area":null},{"id":"2210.17223","title":"Accelerating Distributed MoE Training and Inference with Lina","created":"2022-10-31","cited_by_count":109,"area":"moe"},{"id":"2109.10686","title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","created":"2021-09-22","cited_by_count":106,"area":null},{"id":"2207.10551","title":"Scaling Laws vs Model Architectures: How does Inductive Bias Influence Scaling?","created":"2022-07-21","cited_by_count":103,"area":null},{"id":"2110.12894","title":"The Efficiency Misnomer","created":"2021-10-25","cited_by_count":98,"area":null},{"id":"2109.13226","title":"BigSSL: Exploring the Frontier of Large-Scale Semi-Supervised Learning for Automatic Speech Recognition","created":"2021-09-27","cited_by_count":88,"area":null},{"id":"2102.11972","title":"Do Transformer Modifications Transfer Across Implementations and Applications?","created":"2021-02-23","cited_by_count":87,"area":null},{"id":"2111.12763","title":"Sparse is Enough in Scaling Transformers","created":"2021-11-24","cited_by_count":86,"area":null},{"id":"2203.12533","title":"Pathways: Asynchronous Distributed Dataflow for ML","created":"2022-03-23","cited_by_count":82,"area":null},{"id":"2109.10465","title":"Scalable and Efficient MoE Training for Multitask Multilingual Models","created":"2021-09-22","cited_by_count":76,"area":"moe"},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75,"area":null},{"id":"2203.06311","title":"ELLE: Efficient Lifelong Pre-training for Emerging Data","created":"2022-03-12","cited_by_count":70,"area":null},{"id":"2312.04916","title":"EE-LLM: Large-Scale Training and Inference of Early-Exit Large Language Models with 3D Parallelism","created":"2023-12-08","cited_by_count":69,"area":"inference"},{"id":"2203.00759","title":"HyperPrompt: Prompt-based Task-Conditioning of Transformers","created":"2022-03-01","cited_by_count":67,"area":null},{"id":"2305.12281","title":"Lifelong Language Pretraining with Distribution-Specialized Experts","created":"2023-05-20","cited_by_count":66,"area":"moe"},{"id":"2210.05144","title":"Mixture of Attention Heads: Selecting Attention Heads Per Token","created":"2022-10-11","cited_by_count":60,"area":"moe"},{"id":"2106.11872","title":"Randomness In Neural Network Training: Characterizing The Impact of Tooling","created":"2021-06-22","cited_by_count":59,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2206.04674","title":"Uni-Perceiver-MoE: Learning Sparse Generalist Models with Conditional MoEs","created":"2022-06-09","cited_by_count":58,"area":"moe"},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49,"area":null},{"id":"2209.10797","title":"DFX: A Low-latency Multi-FPGA Appliance for Accelerating Transformer-based Text Generation","created":"2022-09-22","cited_by_count":48,"area":null},{"id":"2107.11906","title":"H-Transformer-1D: Fast One-Dimensional Hierarchical Attention for Sequences","created":"2021-07-25","cited_by_count":47,"area":null},{"id":"2112.12731","title":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-12-23","cited_by_count":46,"area":null},{"id":"2104.08445","title":"Joint Passage Ranking for Diverse Multi-Answer Retrieval","created":"2021-04-17","cited_by_count":43,"area":null},{"id":"2310.16795","title":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","created":"2023-10-25","cited_by_count":42,"area":"moe"},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2105.05720","title":"Breaking the Computation and Communication Abstraction Barrier in Distributed Machine Learning Workloads","created":"2021-05-12","cited_by_count":40,"area":null},{"id":"2105.13880","title":"Knowledge Inheritance for Pre-trained Language Models","created":"2021-05-28","cited_by_count":38,"area":null},{"id":"2303.14676","title":"PDPP: Projected Diffusion for Procedure Planning in Instructional Videos","created":"2023-03-26","cited_by_count":36,"area":null},{"id":"2110.03888","title":"M6-10T: A Sharing-Delinking Paradigm for Efficient Multi-Trillion Parameter Pretraining","created":"2021-10-08","cited_by_count":31,"area":null},{"id":"2405.10853","title":"The Future of Large Language Model Pre-training is Federated","created":"2024-05-17","cited_by_count":30,"area":null},{"id":"2105.03322","title":"Are Pre-trained Convolutions Better than Pre-trained Transformers?","created":"2021-05-07","cited_by_count":27,"area":null},{"id":"2110.08743","title":"GNN-LM: Language Modeling based on Global Contexts via GNN","created":"2021-10-17","cited_by_count":27,"area":null},{"id":"2211.10017","title":"Who Says Elephants Can't Run: Bringing Large Scale MoE Models into Cloud Scale Production","created":"2022-11-18","cited_by_count":27,"area":"moe"},{"id":"2503.09799","title":"Communication-Efficient Language Model Training Scales Reliably and Robustly: Scaling Laws for DiLoCo","created":"2025-03-12","cited_by_count":27,"area":null},{"id":"2308.00852","title":"CASSINI: Network-Aware Job Scheduling in Machine Learning Clusters","created":"2023-08-01","cited_by_count":26,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2305.12129","title":"Lifting the Curse of Capacity Gap in Distilling Language Models","created":"2023-05-20","cited_by_count":25,"area":"moe"},{"id":"2202.07101","title":"A Survey on Dynamic Neural Networks for Natural Language Processing","created":"2022-02-15","cited_by_count":24,"area":null},{"id":"2102.03161","title":"PipeTransformer: Automated Elastic Pipelining for Distributed Training of Transformers","created":"2021-02-05","cited_by_count":23,"area":null},{"id":"2105.14450","title":"Maximizing Parallelism in Distributed Training for Huge Neural Networks","created":"2021-05-30","cited_by_count":23,"area":null},{"id":"2501.04437","title":"Integrating LLMs with ITS: Recent Advances, Potentials, Challenges, and Future Directions","created":"2025-01-08","cited_by_count":23,"area":null},{"id":"2105.15082","title":"M6-T: Exploring Sparse Expert Models and Beyond","created":"2021-05-31","cited_by_count":22,"area":"moe"},{"id":"2205.14336","title":"Gating Dropout: Communication-efficient Regularization for Sparsely Activated Transformers","created":"2022-05-28","cited_by_count":22,"area":"moe"},{"id":"2308.09723","title":"FineQuant: Unlocking Efficiency with Fine-Grained Weight-Only Quantization for LLMs","created":"2023-08-16","cited_by_count":22,"area":"moe"},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20,"area":null},{"id":"2402.06126","title":"Learn To be Efficient: Build Structured Sparsity in Large Language Models","created":"2024-02-09","cited_by_count":20,"area":null},{"id":"2211.05322","title":"On Optimizing the Communication of Model Parallelism","created":"2022-11-10","cited_by_count":19,"area":null},{"id":"2407.05694","title":"On the Limitations of Compute Thresholds as a Governance Strategy","created":"2024-07-08","cited_by_count":18,"area":null},{"id":"2111.05897","title":"Persia: An Open, Hybrid System Scaling Deep Learning-based Recommenders up to 100 Trillion Parameters","created":"2021-11-10","cited_by_count":17,"area":null},{"id":"2206.02107","title":"Interpretable Mixture of Experts","created":"2022-06-05","cited_by_count":17,"area":"moe"},{"id":"2312.02481","title":"Learning to Holistically Detect Bridges from Large-Size VHR Remote Sensing Imagery","created":"2023-12-05","cited_by_count":17,"area":null},{"id":"2108.05818","title":"PatrickStar: Parallel Training of Pre-trained Models via Chunk-based Memory Management","created":"2021-08-12","cited_by_count":15,"area":null},{"id":"2110.13005","title":"AxoNN: An asynchronous, message-driven parallel framework for extreme-scale deep learning","created":"2021-10-25","cited_by_count":15,"area":null},{"id":"2110.15032","title":"OneFlow: Redesign the Distributed Deep Learning Framework from Scratch","created":"2021-10-28","cited_by_count":15,"area":null},{"id":"2302.09915","title":"TA-MoE: Topology-Aware Large Scale Mixture-of-Expert Training","created":"2023-02-20","cited_by_count":15,"area":"moe"},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14,"area":null},{"id":"2107.01499","title":"BAGUA: Scaling up Distributed Learning with System Relaxations","created":"2021-07-03","cited_by_count":13,"area":null},{"id":"2204.01205","title":"Model-Parallel Fourier Neural Operators as Learned Surrogates for Large-Scale Parametric PDEs","created":"2022-04-04","cited_by_count":13,"area":null},{"id":"2205.08808","title":"Evaluation of Transfer Learning for Polish with a Text-to-Text Model","created":"2022-05-18","cited_by_count":13,"area":null},{"id":"2312.11819","title":"An Adaptive Placement and Parallelism Framework for Accelerating RLHF Training","created":"2023-12-19","cited_by_count":13,"area":"alignment"},{"id":"2301.02654","title":"Does compressing activations help model parallel training?","created":"2023-01-06","cited_by_count":12,"area":null},{"id":"2305.12816","title":"Farewell to Aimless Large-scale Pretraining: Influential Subset Selection for Language Model","created":"2023-05-22","cited_by_count":12,"area":null},{"id":"2401.15207","title":"HiFT: A Hierarchical Full Parameter Fine-Tuning Strategy","created":"2024-01-26","cited_by_count":12,"area":null},{"id":"2111.05972","title":"Amazon SageMaker Model Parallelism: A General and Flexible Framework for Large Model Training","created":"2021-11-10","cited_by_count":11,"area":null},{"id":"2508.00053","title":"A Quality-Guided Mixture of Score-Fusion Experts Framework for Human Recognition","created":"2025-07-31","cited_by_count":11,"area":null},{"id":"2312.02520","title":"Towards More Unified In-context Visual Understanding","created":"2023-12-05","cited_by_count":10,"area":null},{"id":"2401.11202","title":"PartIR: Composing SPMD Partitioning Strategies for Machine Learning","created":"2024-01-20","cited_by_count":10,"area":null},{"id":"2204.11458","title":"ED2LM: Encoder-Decoder to Language Model for Faster Document Re-ranking Inference","created":"2022-04-25","cited_by_count":9,"area":null},{"id":"2403.07128","title":"DrJAX: Scalable and Differentiable MapReduce Primitives in JAX","created":"2024-03-11","cited_by_count":9,"area":null},{"id":"2406.17145","title":"GraphPipe: Improving Performance and Scalability of DNN Training with Graph Pipeline Parallelism","created":"2024-06-24","cited_by_count":9,"area":null},{"id":"2112.01637","title":"AdaSplit: Adaptive Trade-offs for Resource-constrained Distributed Deep Learning","created":"2021-12-02","cited_by_count":8,"area":null},{"id":"2411.05614","title":"Acceleration for Deep Reinforcement Learning using Parallel and Distributed Computing: A Survey","created":"2024-11-08","cited_by_count":8,"area":"deeprl"},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null}],"next_cursor":"100"}