{"total":129,"rows":[{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326,"area":null},{"id":"2208.07339","title":"LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale","created":"2022-08-15","cited_by_count":906,"area":null},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535,"area":null},{"id":"2501.13928","title":"Fast3R: Towards 3D Reconstruction of 1000+ Images in One Forward Pass","created":"2025-01-23","cited_by_count":367,"area":null},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2104.08378","title":"Accelerating Sparse Deep Neural Networks","created":"2021-04-16","cited_by_count":223,"area":null},{"id":"2203.17189","title":"Scaling Up Models and Data with $\\texttt{t5x}$ and $\\texttt{seqio}$","created":"2022-03-31","cited_by_count":187,"area":null},{"id":"2202.11176","title":"A New Generation of Perspective API: Efficient Multilingual Character-level Transformers","created":"2022-02-22","cited_by_count":185,"area":null},{"id":"2111.10050","title":"Combined Scaling for Zero-shot Transfer Learning","created":"2021-11-19","cited_by_count":182,"area":null},{"id":"2111.10952","title":"ExT5: Towards Extreme Multi-Task Scaling for Transfer Learning","created":"2021-11-22","cited_by_count":161,"area":null},{"id":"2106.12672","title":"Charformer: Fast Character Transformers via Gradient-based Subword Tokenization","created":"2021-06-23","cited_by_count":160,"area":null},{"id":"2104.12369","title":"PanGu-$\\alpha$: Large-scale Autoregressive Pretrained Chinese Language Models with Auto-parallel Computation","created":"2021-04-26","cited_by_count":134,"area":null},{"id":"2112.03052","title":"Scaling Up Influence Functions","created":"2021-12-06","cited_by_count":132,"area":null},{"id":"2105.04663","title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","created":"2021-05-10","cited_by_count":111,"area":null},{"id":"2109.10686","title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","created":"2021-09-22","cited_by_count":106,"area":null},{"id":"2207.10551","title":"Scaling Laws vs Model Architectures: How does Inductive Bias Influence Scaling?","created":"2022-07-21","cited_by_count":103,"area":null},{"id":"2110.12894","title":"The Efficiency Misnomer","created":"2021-10-25","cited_by_count":98,"area":null},{"id":"2109.13226","title":"BigSSL: Exploring the Frontier of Large-Scale Semi-Supervised Learning for Automatic Speech Recognition","created":"2021-09-27","cited_by_count":88,"area":null},{"id":"2102.11972","title":"Do Transformer Modifications Transfer Across Implementations and Applications?","created":"2021-02-23","cited_by_count":87,"area":null},{"id":"2111.12763","title":"Sparse is Enough in Scaling Transformers","created":"2021-11-24","cited_by_count":86,"area":null},{"id":"2203.12533","title":"Pathways: Asynchronous Distributed Dataflow for ML","created":"2022-03-23","cited_by_count":82,"area":null},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75,"area":null},{"id":"2203.06311","title":"ELLE: Efficient Lifelong Pre-training for Emerging Data","created":"2022-03-12","cited_by_count":70,"area":null},{"id":"2203.00759","title":"HyperPrompt: Prompt-based Task-Conditioning of Transformers","created":"2022-03-01","cited_by_count":67,"area":null},{"id":"2106.11872","title":"Randomness In Neural Network Training: Characterizing The Impact of Tooling","created":"2021-06-22","cited_by_count":59,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49,"area":null},{"id":"2209.10797","title":"DFX: A Low-latency Multi-FPGA Appliance for Accelerating Transformer-based Text Generation","created":"2022-09-22","cited_by_count":48,"area":null},{"id":"2107.11906","title":"H-Transformer-1D: Fast One-Dimensional Hierarchical Attention for Sequences","created":"2021-07-25","cited_by_count":47,"area":null},{"id":"2112.12731","title":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-12-23","cited_by_count":46,"area":null},{"id":"2104.08445","title":"Joint Passage Ranking for Diverse Multi-Answer Retrieval","created":"2021-04-17","cited_by_count":43,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2105.05720","title":"Breaking the Computation and Communication Abstraction Barrier in Distributed Machine Learning Workloads","created":"2021-05-12","cited_by_count":40,"area":null},{"id":"2105.13880","title":"Knowledge Inheritance for Pre-trained Language Models","created":"2021-05-28","cited_by_count":38,"area":null},{"id":"2303.14676","title":"PDPP: Projected Diffusion for Procedure Planning in Instructional Videos","created":"2023-03-26","cited_by_count":36,"area":null},{"id":"2110.03888","title":"M6-10T: A Sharing-Delinking Paradigm for Efficient Multi-Trillion Parameter Pretraining","created":"2021-10-08","cited_by_count":31,"area":null},{"id":"2405.10853","title":"The Future of Large Language Model Pre-training is Federated","created":"2024-05-17","cited_by_count":30,"area":null},{"id":"2105.03322","title":"Are Pre-trained Convolutions Better than Pre-trained Transformers?","created":"2021-05-07","cited_by_count":27,"area":null},{"id":"2110.08743","title":"GNN-LM: Language Modeling based on Global Contexts via GNN","created":"2021-10-17","cited_by_count":27,"area":null},{"id":"2503.09799","title":"Communication-Efficient Language Model Training Scales Reliably and Robustly: Scaling Laws for DiLoCo","created":"2025-03-12","cited_by_count":27,"area":null},{"id":"2308.00852","title":"CASSINI: Network-Aware Job Scheduling in Machine Learning Clusters","created":"2023-08-01","cited_by_count":26,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2202.07101","title":"A Survey on Dynamic Neural Networks for Natural Language Processing","created":"2022-02-15","cited_by_count":24,"area":null},{"id":"2102.03161","title":"PipeTransformer: Automated Elastic Pipelining for Distributed Training of Transformers","created":"2021-02-05","cited_by_count":23,"area":null},{"id":"2105.14450","title":"Maximizing Parallelism in Distributed Training for Huge Neural Networks","created":"2021-05-30","cited_by_count":23,"area":null},{"id":"2501.04437","title":"Integrating LLMs with ITS: Recent Advances, Potentials, Challenges, and Future Directions","created":"2025-01-08","cited_by_count":23,"area":null},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20,"area":null},{"id":"2402.06126","title":"Learn To be Efficient: Build Structured Sparsity in Large Language Models","created":"2024-02-09","cited_by_count":20,"area":null},{"id":"2211.05322","title":"On Optimizing the Communication of Model Parallelism","created":"2022-11-10","cited_by_count":19,"area":null},{"id":"2407.05694","title":"On the Limitations of Compute Thresholds as a Governance Strategy","created":"2024-07-08","cited_by_count":18,"area":null},{"id":"2111.05897","title":"Persia: An Open, Hybrid System Scaling Deep Learning-based Recommenders up to 100 Trillion Parameters","created":"2021-11-10","cited_by_count":17,"area":null},{"id":"2312.02481","title":"Learning to Holistically Detect Bridges from Large-Size VHR Remote Sensing Imagery","created":"2023-12-05","cited_by_count":17,"area":null},{"id":"2108.05818","title":"PatrickStar: Parallel Training of Pre-trained Models via Chunk-based Memory Management","created":"2021-08-12","cited_by_count":15,"area":null},{"id":"2110.13005","title":"AxoNN: An asynchronous, message-driven parallel framework for extreme-scale deep learning","created":"2021-10-25","cited_by_count":15,"area":null},{"id":"2110.15032","title":"OneFlow: Redesign the Distributed Deep Learning Framework from Scratch","created":"2021-10-28","cited_by_count":15,"area":null},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14,"area":null},{"id":"2107.01499","title":"BAGUA: Scaling up Distributed Learning with System Relaxations","created":"2021-07-03","cited_by_count":13,"area":null},{"id":"2204.01205","title":"Model-Parallel Fourier Neural Operators as Learned Surrogates for Large-Scale Parametric PDEs","created":"2022-04-04","cited_by_count":13,"area":null},{"id":"2205.08808","title":"Evaluation of Transfer Learning for Polish with a Text-to-Text Model","created":"2022-05-18","cited_by_count":13,"area":null},{"id":"2301.02654","title":"Does compressing activations help model parallel training?","created":"2023-01-06","cited_by_count":12,"area":null},{"id":"2305.12816","title":"Farewell to Aimless Large-scale Pretraining: Influential Subset Selection for Language Model","created":"2023-05-22","cited_by_count":12,"area":null},{"id":"2401.15207","title":"HiFT: A Hierarchical Full Parameter Fine-Tuning Strategy","created":"2024-01-26","cited_by_count":12,"area":null},{"id":"2111.05972","title":"Amazon SageMaker Model Parallelism: A General and Flexible Framework for Large Model Training","created":"2021-11-10","cited_by_count":11,"area":null},{"id":"2508.00053","title":"A Quality-Guided Mixture of Score-Fusion Experts Framework for Human Recognition","created":"2025-07-31","cited_by_count":11,"area":null},{"id":"2312.02520","title":"Towards More Unified In-context Visual Understanding","created":"2023-12-05","cited_by_count":10,"area":null},{"id":"2401.11202","title":"PartIR: Composing SPMD Partitioning Strategies for Machine Learning","created":"2024-01-20","cited_by_count":10,"area":null},{"id":"2204.11458","title":"ED2LM: Encoder-Decoder to Language Model for Faster Document Re-ranking Inference","created":"2022-04-25","cited_by_count":9,"area":null},{"id":"2403.07128","title":"DrJAX: Scalable and Differentiable MapReduce Primitives in JAX","created":"2024-03-11","cited_by_count":9,"area":null},{"id":"2406.17145","title":"GraphPipe: Improving Performance and Scalability of DNN Training with Graph Pipeline Parallelism","created":"2024-06-24","cited_by_count":9,"area":null},{"id":"2112.01637","title":"AdaSplit: Adaptive Trade-offs for Resource-constrained Distributed Deep Learning","created":"2021-12-02","cited_by_count":8,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2103.16063","title":"Automatic Graph Partitioning for Very Large-scale Deep Learning","created":"2021-03-30","cited_by_count":7,"area":null},{"id":"2110.03848","title":"Speeding up Deep Model Training by Sharing Weights and Then Unsharing","created":"2021-10-08","cited_by_count":7,"area":null},{"id":"2110.08426","title":"EncT5: A Framework for Fine-tuning T5 as Non-autoregressive Models","created":"2021-10-16","cited_by_count":7,"area":null},{"id":"2111.04949","title":"A Survey and Empirical Evaluation of Parallel Deep Learning Frameworks","created":"2021-11-09","cited_by_count":7,"area":null},{"id":"2403.03699","title":"Model Parallelism on Distributed Infrastructure: A Literature Review from Theory to LLM Case-Studies","created":"2024-03-06","cited_by_count":7,"area":null},{"id":"2106.03640","title":"Making EfficientNet More Efficient: Exploring Batch-Independent Normalization, Group Convolutions and Reduced Resolution Training","created":"2021-06-07","cited_by_count":6,"area":null},{"id":"2410.23261","title":"$100K or 100 Days: Trade-offs when Pre-Training with Academic Resources","created":"2024-10-30","cited_by_count":6,"area":null},{"id":"2102.01963","title":"Local Critic Training for Model-Parallel Learning of Deep Neural Networks","created":"2021-02-03","cited_by_count":5,"area":null},{"id":"2112.02752","title":"End-to-end Adaptive Distributed Training on PaddlePaddle","created":"2021-12-06","cited_by_count":5,"area":null},{"id":"2301.08984","title":"SuperScaler: Supporting Flexible DNN Parallelization via a Unified Abstraction","created":"2023-01-21","cited_by_count":5,"area":null},{"id":"2408.04808","title":"Scaling Deep Learning Computation over the Inter-Core Connected Intelligence Processor with T10","created":"2024-08-09","cited_by_count":5,"area":null},{"id":"2206.14148","title":"Memory Safe Computations with XLA Compiler","created":"2022-06-28","cited_by_count":4,"area":null},{"id":"2301.02691","title":"Systems for Parallel and Distributed Large-Model Deep Learning Training","created":"2023-01-06","cited_by_count":4,"area":null},{"id":"2501.14784","title":"DeServe: Towards Affordable Offline LLM Inference via Decentralization","created":"2025-01-04","cited_by_count":4,"area":null},{"id":"2604.21428","title":"Decoupled DiLoCo for Resilient Distributed Pre-training","created":"2026-04-23","cited_by_count":4,"area":null},{"id":"2105.14156","title":"SMASH: Sparse Matrix Atomic Scratchpad Hashing","created":"2021-05-29","cited_by_count":3,"area":null},{"id":"2112.01075","title":"Memory-efficient array redistribution through portable collective communication","created":"2021-12-02","cited_by_count":3,"area":null},{"id":"2205.09470","title":"Nebula-I: A General Framework for Collaboratively Training Deep Learning Models on Low-Bandwidth Cloud Clusters","created":"2022-05-19","cited_by_count":3,"area":null},{"id":"2211.05953","title":"Breadth-First Pipeline Parallelism","created":"2022-11-11","cited_by_count":3,"area":null},{"id":"2302.00247","title":"TAPAS: Fast and Automatic Derivation of Tensor Parallel Strategies for Large Neural Networks","created":"2023-02-01","cited_by_count":3,"area":null},{"id":"2502.19790","title":"Mixtera: A Data Plane for Foundation Model Training","created":"2025-02-27","cited_by_count":3,"area":null},{"id":"2507.09029","title":"Model Parallelism With Subnetwork Data Parallelism","created":"2025-07-11","cited_by_count":3,"area":null},{"id":"2601.19092","title":"Axe: A Simple Unified Layout Abstraction for Machine Learning Compilers","created":"2026-01-27","cited_by_count":3,"area":null},{"id":"2112.04041","title":"A Transferable Approach for Partitioning Machine Learning Models on Multi-Chip-Modules","created":"2021-12-07","cited_by_count":2,"area":null},{"id":"2211.16648","title":"COMET: A Comprehensive Cluster Design Methodology for Distributed Deep Learning Training","created":"2022-11-30","cited_by_count":2,"area":null}],"next_cursor":"100"}