{"total":157,"rows":[{"id":"2305.05176","title":"FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance","created":"2023-05-09","cited_by_count":853,"area":null},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660,"area":null},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2104.12369","title":"PanGu-$\\alpha$: Large-scale Autoregressive Pretrained Chinese Language Models with Auto-parallel Computation","created":"2021-04-26","cited_by_count":134,"area":null},{"id":"2105.04663","title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","created":"2021-05-10","cited_by_count":111,"area":null},{"id":"2302.14017","title":"Full Stack Optimization of Transformer Inference: a Survey","created":"2023-02-27","cited_by_count":80,"area":null},{"id":"2104.05158","title":"Software-Hardware Co-design for Fast and Scalable Training of Deep Learning Recommendation Models","created":"2021-04-12","cited_by_count":78,"area":null},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75,"area":null},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53,"area":null},{"id":"2105.11118","title":"Dorylus: Affordable, Scalable, and Accurate GNN Training with Distributed CPU Servers and Serverless Threads","created":"2021-05-24","cited_by_count":51,"area":null},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49,"area":null},{"id":"2303.14006","title":"ASTRA-sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-model Training at Scale","created":"2023-03-24","cited_by_count":47,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2405.05751","title":"Mirage: A Multi-Level Superoptimizer for Tensor Programs","created":"2024-05-09","cited_by_count":41,"area":null},{"id":"2105.05720","title":"Breaking the Computation and Communication Abstraction Barrier in Distributed Machine Learning Workloads","created":"2021-05-12","cited_by_count":40,"area":null},{"id":"2107.06419","title":"FLAT: An Optimized Dataflow for Mitigating Attention Bottlenecks","created":"2021-07-13","cited_by_count":39,"area":null},{"id":"2204.11786","title":"Enable Deep Learning on Mobile Devices: Methods, Systems, and Applications","created":"2022-04-25","cited_by_count":32,"area":null},{"id":"2211.00216","title":"Distributed Graph Neural Network Training: A Survey","created":"2022-11-01","cited_by_count":32,"area":null},{"id":"2405.10313","title":"How Far Are We From AGI: Are LLMs All We Need?","created":"2024-05-16","cited_by_count":31,"area":null},{"id":"2102.00527","title":"A Runtime-Based Computational Performance Predictor for Deep Neural Network Training","created":"2021-01-31","cited_by_count":29,"area":null},{"id":"2103.00543","title":"On the Utility of Gradient Compression in Distributed Training Systems","created":"2021-02-28","cited_by_count":26,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2201.12465","title":"Flashlight: Enabling Innovation in Tools for Machine Learning","created":"2022-01-29","cited_by_count":25,"area":null},{"id":"2203.11201","title":"Efficient Neural Network Analysis with Sum-of-Infeasibilities","created":"2022-03-19","cited_by_count":24,"area":null},{"id":"2312.12391","title":"vTrain: A Simulation Framework for Evaluating Cost-effective and Compute-optimal Large Language Model Training","created":"2023-11-27","cited_by_count":23,"area":null},{"id":"2103.00686","title":"Accelerating Recommendation System Training by Leveraging Popular Choices","created":"2021-03-01","cited_by_count":22,"area":null},{"id":"2105.01898","title":"CoSA: Scheduling by Constrained Optimization for Spatial Accelerators","created":"2021-05-05","cited_by_count":22,"area":null},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20,"area":null},{"id":"2211.05322","title":"On Optimizing the Communication of Model Parallelism","created":"2022-11-10","cited_by_count":19,"area":null},{"id":"2412.01523","title":"FlexSP: Accelerating Large Language Model Training via Flexible Sequence Parallelism","created":"2024-12-02","cited_by_count":19,"area":null},{"id":"2104.05245","title":"Distributed Learning Systems with First-order Methods","created":"2021-04-12","cited_by_count":17,"area":null},{"id":"2111.05897","title":"Persia: An Open, Hybrid System Scaling Deep Learning-based Recommenders up to 100 Trillion Parameters","created":"2021-11-10","cited_by_count":17,"area":null},{"id":"2309.10285","title":"Flash-LLM: Enabling Cost-Effective and Highly-Efficient Large Generative Model Inference with Unstructured Sparsity","created":"2023-09-19","cited_by_count":17,"area":null},{"id":"2312.05181","title":"Tenplex: Dynamic Parallelism for Deep Learning using Parallelizable Tensor Collections","created":"2023-12-08","cited_by_count":16,"area":null},{"id":"2110.15032","title":"OneFlow: Redesign the Distributed Deep Learning Framework from Scratch","created":"2021-10-28","cited_by_count":15,"area":null},{"id":"2301.09830","title":"Optimus-CC: Efficient Large NLP Model Training with 3D Parallelism Aware Communication Compression","created":"2023-01-24","cited_by_count":15,"area":null},{"id":"2210.02023","title":"DreamShard: Generalizable Embedding Table Placement for Recommender Systems","created":"2022-10-05","cited_by_count":14,"area":null},{"id":"2302.03225","title":"Efficient XAI Techniques: A Taxonomic Survey","created":"2023-02-07","cited_by_count":14,"area":null},{"id":"2307.02031","title":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","created":"2023-07-05","cited_by_count":14,"area":null},{"id":"2310.02784","title":"MAD Max Beyond Single-Node: Enabling Large Machine Learning Model Acceleration on Distributed Systems","created":"2023-10-04","cited_by_count":14,"area":null},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14,"area":null},{"id":"2107.01499","title":"BAGUA: Scaling up Distributed Learning with System Relaxations","created":"2021-07-03","cited_by_count":13,"area":null},{"id":"2210.07297","title":"AMP: Automatically Finding Model Parallel Strategies with Heterogeneity Awareness","created":"2022-10-13","cited_by_count":13,"area":null},{"id":"2305.14516","title":"Chakra: Advancing Performance Benchmarking and Co-design using Standardized Execution Traces","created":"2023-05-23","cited_by_count":13,"area":null},{"id":"2112.02958","title":"Automap: Towards Ergonomic Automated Parallelism for ML Models","created":"2021-12-06","cited_by_count":12,"area":null},{"id":"2205.02473","title":"dPRO: A Generic Profiling and Optimization System for Expediting Distributed DNN Training","created":"2022-05-05","cited_by_count":12,"area":null},{"id":"2301.02654","title":"Does compressing activations help model parallel training?","created":"2023-01-06","cited_by_count":12,"area":null},{"id":"2312.02515","title":"mLoRA: Fine-Tuning LoRA Adapters via Highly-Efficient Pipeline Parallelism in Multiple GPUs","created":"2023-12-05","cited_by_count":12,"area":null},{"id":"2202.08679","title":"Where Is My Training Bottleneck? Hidden Trade-Offs in Deep Learning Preprocessing Pipelines","created":"2022-02-17","cited_by_count":10,"area":null},{"id":"2202.12429","title":"BagPipe: Accelerating Deep Recommendation Model Training","created":"2022-02-24","cited_by_count":10,"area":null},{"id":"2401.11202","title":"PartIR: Composing SPMD Partitioning Strategies for Machine Learning","created":"2024-01-20","cited_by_count":10,"area":null},{"id":"2409.11416","title":"The Unseen AI Disruptions for Power Grids: LLM-Induced Transients","created":"2024-09-09","cited_by_count":10,"area":null},{"id":"2403.07128","title":"DrJAX: Scalable and Differentiable MapReduce Primitives in JAX","created":"2024-03-11","cited_by_count":9,"area":null},{"id":"2406.17145","title":"GraphPipe: Improving Performance and Scalability of DNN Training with Graph Pipeline Parallelism","created":"2024-06-24","cited_by_count":9,"area":null},{"id":"2209.01346","title":"HammingMesh: A Network Topology for Large-Scale Deep Learning","created":"2022-09-03","cited_by_count":8,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2103.16063","title":"Automatic Graph Partitioning for Very Large-scale Deep Learning","created":"2021-03-30","cited_by_count":7,"area":null},{"id":"2111.04949","title":"A Survey and Empirical Evaluation of Parallel Deep Learning Frameworks","created":"2021-11-09","cited_by_count":7,"area":null},{"id":"2403.03699","title":"Model Parallelism on Distributed Infrastructure: A Literature Review from Theory to LLM Case-Studies","created":"2024-03-06","cited_by_count":7,"area":null},{"id":"2409.03365","title":"Spindle: Efficient Distributed Training of Multi-Task Large Models via Wavefront Scheduling","created":"2024-09-05","cited_by_count":7,"area":null},{"id":"2107.06469","title":"Model-Parallel Model Selection for Deep Learning Systems","created":"2021-07-14","cited_by_count":6,"area":null},{"id":"2110.14007","title":"TOD: GPU-accelerated Outlier Detection via Tensor Operations","created":"2021-10-26","cited_by_count":6,"area":null},{"id":"2102.01963","title":"Local Critic Training for Model-Parallel Learning of Deep Neural Networks","created":"2021-02-03","cited_by_count":5,"area":null},{"id":"2112.02752","title":"End-to-end Adaptive Distributed Training on PaddlePaddle","created":"2021-12-06","cited_by_count":5,"area":null},{"id":"2202.03356","title":"Efficient Direct-Connect Topologies for Collective Communications","created":"2022-02-07","cited_by_count":5,"area":null},{"id":"2301.08984","title":"SuperScaler: Supporting Flexible DNN Parallelization via a Unified Abstraction","created":"2023-01-21","cited_by_count":5,"area":null},{"id":"2302.02599","title":"Colossal-Auto: Unified Automation of Parallelization and Activation Checkpoint for Large-scale Models","created":"2023-02-06","cited_by_count":5,"area":null},{"id":"2305.01868","title":"Pre-train and Search: Efficient Embedding Table Sharding with Pre-trained Neural Cost Models","created":"2023-05-03","cited_by_count":5,"area":null},{"id":"2408.04808","title":"Scaling Deep Learning Computation over the Inter-Core Connected Intelligence Processor with T10","created":"2024-08-09","cited_by_count":5,"area":null},{"id":"2410.07192","title":"PipeFill: Using GPUs During Bubbles in Pipeline-parallel LLM Training","created":"2024-09-23","cited_by_count":5,"area":null},{"id":"2509.10371","title":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","created":"2025-09-12","cited_by_count":5,"area":null},{"id":"2104.11805","title":"Partitioning sparse deep neural networks for scalable training and inference","created":"2021-04-23","cited_by_count":4,"area":null},{"id":"2110.10548","title":"Synthesizing Optimal Parallelism Placement and Reduction Strategies on Hierarchical Systems for Deep Learning","created":"2021-10-20","cited_by_count":4,"area":null},{"id":"2205.03960","title":"Neural Architecture Search using Property Guided Synthesis","created":"2022-05-08","cited_by_count":4,"area":null},{"id":"2301.02691","title":"Systems for Parallel and Distributed Large-Model Deep Learning Training","created":"2023-01-06","cited_by_count":4,"area":null},{"id":"2407.13143","title":"Integrated Hardware Architecture and Device Placement Search","created":"2024-07-18","cited_by_count":4,"area":null},{"id":"2110.08633","title":"Hydra: A System for Large Multi-Model Deep Learning","created":"2021-10-16","cited_by_count":3,"area":null},{"id":"2110.12484","title":"Enabling Large Batch Size Training for DNN Models Beyond the Memory Limit While Maintaining Performance","created":"2021-10-24","cited_by_count":3,"area":null},{"id":"2112.01075","title":"Memory-efficient array redistribution through portable collective communication","created":"2021-12-02","cited_by_count":3,"area":null},{"id":"2112.10065","title":"Efficient Strong Scaling Through Burst Parallel Training","created":"2021-12-19","cited_by_count":3,"area":null},{"id":"2208.11879","title":"A simplified convergence theory for Byzantine resilient stochastic gradient descent","created":"2022-08-25","cited_by_count":3,"area":null},{"id":"2211.17246","title":"Pex: Memory-efficient Microcontroller Deep Learning through Partial Execution","created":"2022-11-30","cited_by_count":3,"area":null},{"id":"2302.00247","title":"TAPAS: Fast and Automatic Derivation of Tensor Parallel Strategies for Large Neural Networks","created":"2023-02-01","cited_by_count":3,"area":null},{"id":"2308.08500","title":"InTune: Reinforcement Learning-based Data Pipeline Optimization for Deep Recommendation Models","created":"2023-08-13","cited_by_count":3,"area":null},{"id":"2309.16743","title":"High Throughput Training of Deep Surrogates from Large Ensemble Runs","created":"2023-09-28","cited_by_count":3,"area":null},{"id":"2311.00591","title":"Coop: Memory is not a Commodity","created":"2023-11-01","cited_by_count":3,"area":null},{"id":"2311.09690","title":"CDMPP: A Device-Model Agnostic Framework for Latency Prediction of Tensor Programs","created":"2023-11-16","cited_by_count":3,"area":null},{"id":"2402.00208","title":"MP-SL: Multihop Parallel Split Learning","created":"2024-01-31","cited_by_count":3,"area":null},{"id":"2412.07894","title":"Hydraulis: Balancing Large Transformer Model Training via Co-designing Parallel Strategies and Data Assignment","created":"2024-12-10","cited_by_count":3,"area":null},{"id":"2507.21276","title":"LeMix: Unified Scheduling for LLM Training and Inference on Multi-GPU Systems","created":"2025-07-28","cited_by_count":3,"area":null},{"id":"2601.12698","title":"A Two-Stage GPU Kernel Tuner Combining Semantic Refactoring and Search-Based Optimization","created":"2026-01-19","cited_by_count":3,"area":null},{"id":"2601.19092","title":"Axe: A Simple Unified Layout Abstraction for Machine Learning Compilers","created":"2026-01-27","cited_by_count":3,"area":null},{"id":"2104.09075","title":"An Oracle for Guiding Large-Scale Model/Hybrid Parallel Training of Convolutional Neural Networks","created":"2021-04-19","cited_by_count":2,"area":null},{"id":"2107.06960","title":"MAFAT: Memory-Aware Fusing and Tiling of Neural Networks for Accelerated Edge Inference","created":"2021-07-14","cited_by_count":2,"area":null},{"id":"2110.15884","title":"Distributing Deep Learning Hyperparameter Tuning for 3D Medical Image Segmentation","created":"2021-10-29","cited_by_count":2,"area":null},{"id":"2111.00655","title":"Collage: Seamless Integration of Deep Learning Backends with Automatic Placement","created":"2021-11-01","cited_by_count":2,"area":null}],"next_cursor":"100"}