{"total":53,"rows":[{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2203.12533","title":"Pathways: Asynchronous Distributed Dataflow for ML","created":"2022-03-23","cited_by_count":82,"area":null},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2406.06858","title":"FLUX: Fast Software-based Communication Overlap On GPUs Through Kernel Fusion","created":"2024-06-11","cited_by_count":57,"area":null},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53,"area":null},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49,"area":null},{"id":"2203.10428","title":"PipeGCN: Efficient Full-Graph Training of Graph Convolutional Networks with Pipelined Feature Communication","created":"2022-03-20","cited_by_count":45,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2401.10241","title":"Zero Bubble Pipeline Parallelism","created":"2023-11-30","cited_by_count":40,"area":null},{"id":"2312.15159","title":"Understanding the Potential of FPGA-Based Spatial Acceleration for Large Language Model Inference","created":"2023-12-23","cited_by_count":33,"area":null},{"id":"2206.01299","title":"Fine-tuning Language Models over Slow Networks using Activation Compression with Guarantees","created":"2022-06-02","cited_by_count":30,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2201.06227","title":"Egeria: Efficient DNN Training with Knowledge-Guided Layer Freezing","created":"2022-01-17","cited_by_count":22,"area":null},{"id":"2110.13005","title":"AxoNN: An asynchronous, message-driven parallel framework for extreme-scale deep learning","created":"2021-10-25","cited_by_count":15,"area":null},{"id":"2301.09830","title":"Optimus-CC: Efficient Large NLP Model Training with 3D Parallelism Aware Communication Compression","created":"2023-01-24","cited_by_count":15,"area":null},{"id":"2403.06664","title":"Smart-Infinity: Fast Large Language Model Training using Near-Storage Processing on a Real System","created":"2024-03-11","cited_by_count":15,"area":null},{"id":"2307.02031","title":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","created":"2023-07-05","cited_by_count":14,"area":null},{"id":"2110.14895","title":"Pipeline Parallelism for Inference on Heterogeneous Edge Computing","created":"2021-10-28","cited_by_count":13,"area":null},{"id":"2112.02958","title":"Automap: Towards Ergonomic Automated Parallelism for ML Models","created":"2021-12-06","cited_by_count":12,"area":null},{"id":"2312.02515","title":"mLoRA: Fine-Tuning LoRA Adapters via Highly-Efficient Pipeline Parallelism in Multiple GPUs","created":"2023-12-05","cited_by_count":12,"area":null},{"id":"2206.04959","title":"Merak: An Efficient Distributed DNN Training Framework with Automated 3D Parallelism for Giant Foundation Models","created":"2022-06-10","cited_by_count":11,"area":null},{"id":"2405.15362","title":"Pipeline Parallelism with Controllable Memory","created":"2024-05-24","cited_by_count":11,"area":null},{"id":"2209.01346","title":"HammingMesh: A Network Topology for Large-Scale Deep Learning","created":"2022-09-03","cited_by_count":8,"area":null},{"id":"2206.14098","title":"RevBiFPN: The Fully Reversible Bidirectional Feature Pyramid Network","created":"2022-06-28","cited_by_count":6,"area":null},{"id":"2406.02052","title":"PETRA: Parallel End-to-end Training with Reversible Architectures","created":"2024-06-04","cited_by_count":6,"area":null},{"id":"2406.02613","title":"ACCO: Accumulate While You Communicate for Communication-Overlapped Sharded LLM Training","created":"2024-06-03","cited_by_count":6,"area":null},{"id":"2505.01099","title":"Nesterov Method for Asynchronous Pipeline Parallel Optimization","created":"2025-05-02","cited_by_count":6,"area":null},{"id":"2302.05045","title":"Exploiting Sparsity in Pruned Neural Networks to Optimize Large Model Training","created":"2023-02-10","cited_by_count":5,"area":null},{"id":"2103.08802","title":"Parareal Neural Networks Emulating a Parallel-in-time Algorithm","created":"2021-03-16","cited_by_count":4,"area":null},{"id":"2202.10435","title":"Survey on Large Scale Neural Network Training","created":"2022-02-21","cited_by_count":4,"area":null},{"id":"2301.02691","title":"Systems for Parallel and Distributed Large-Model Deep Learning Training","created":"2023-01-06","cited_by_count":4,"area":null},{"id":"2312.00839","title":"PipeOptim: Ensuring Effective 1F1B Schedule with Optimizer-Dependent Weight Prediction","created":"2023-12-01","cited_by_count":4,"area":null},{"id":"2110.08633","title":"Hydra: A System for Large Multi-Model Deep Learning","created":"2021-10-16","cited_by_count":3,"area":null},{"id":"2601.22442","title":"AsyncMesh: Fully Asynchronous Optimization for Data and Pipeline Parallelism","created":"2026-01-30","cited_by_count":3,"area":null},{"id":"2207.11019","title":"Layer-Wise Partitioning and Merging for Efficient and Scalable Deep Learning","created":"2022-07-22","cited_by_count":2,"area":null},{"id":"2210.06352","title":"Automatic Discovery of Composite SPMD Partitioning Strategies in PartIR","created":"2022-10-07","cited_by_count":2,"area":null},{"id":"2301.08695","title":"Baechi: Fast Device Placement of Machine Learning Graphs","created":"2023-01-20","cited_by_count":2,"area":null},{"id":"2302.12803","title":"PiPar: Pipeline Parallelism for Collaborative Machine Learning","created":"2022-12-01","cited_by_count":2,"area":null},{"id":"2309.01226","title":"Saturn: An Optimized Data System for Large Model Deep Learning Workloads","created":"2023-09-03","cited_by_count":2,"area":null},{"id":"2401.01728","title":"Ravnest: Decentralized Asynchronous Training on Heterogeneous Devices","created":"2024-01-03","cited_by_count":2,"area":null},{"id":"2602.03515","title":"Mitigating Staleness in Asynchronous Pipeline Parallelism via Basis Rotation","created":"2026-02-03","cited_by_count":2,"area":null},{"id":"2312.00843","title":"Exploring the Robustness of Decentralized Training for Large Language Models","created":"2023-12-01","cited_by_count":1,"area":null},{"id":"2501.09849","title":"Coded Deep Learning: Framework and Algorithm","created":"2025-01-16","cited_by_count":1,"area":null},{"id":"2311.03703","title":"Practical Performance Guarantees for Pipelined DNN Inference","created":"2023-11-07","cited_by_count":0,"area":null},{"id":"2403.08837","title":"Cyclic Data Parallelism for Efficient Parallelism of Deep Neural Networks","created":"2024-03-13","cited_by_count":0,"area":null},{"id":"2405.18047","title":"2BP: 2-Stage Backpropagation","created":"2024-05-28","cited_by_count":0,"area":null},{"id":"2512.21884","title":"Optimizing Resource Allocation for Geographically-Distributed Inference by Large Language Models","created":"2025-12-26","cited_by_count":0,"area":null},{"id":"2604.27085","title":"Efficient Training on Multiple Consumer GPUs with RoundPipe","created":"2026-04-29","cited_by_count":0,"area":null},{"id":"2605.18750","title":"A Readiness-Driven Runtime for Pipeline-Parallel Training under Runtime Variability","created":"2026-05-18","cited_by_count":0,"area":null},{"id":"2606.07881","title":"Breaking the Bubble: Asynchronous Pipeline Parallel Training with Bounded Weight Inconsistency","created":"2026-06-05","cited_by_count":0,"area":null},{"id":"2606.30634","title":"One-Step Gradient Delay is Not a Barrier for Large-Scale Asynchronous Pipeline Parallel LLM Pretraining","created":"2026-06-29","cited_by_count":0,"area":null},{"id":"2610.14935","title":"BSIPipe: Bidirectional Sequence-Level Interleaved Pipeline Parallelism for Long-Sequence Large Language Model Training","created":"2026-10-10","cited_by_count":0,"area":null}],"next_cursor":null}