{"total":116,"rows":[{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326,"area":null},{"id":"2210.02414","title":"GLM-130B: An Open Bilingual Pre-trained Model","created":"2022-10-05","cited_by_count":843,"area":null},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535,"area":null},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245,"area":null},{"id":"2404.17790","title":"Continual Pre-Training for Cross-Lingual LLM Adaptation: Enhancing Japanese Language Capabilities","created":"2024-04-27","cited_by_count":111,"area":null},{"id":"2206.01288","title":"Decentralized Training of Foundation Models in Heterogeneous Environments","created":"2022-06-02","cited_by_count":87,"area":null},{"id":"2512.16776","title":"Kling-Omni Technical Report","created":"2025-12-18","cited_by_count":71,"area":null},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68,"area":null},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53,"area":null},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52,"area":null},{"id":"2401.06199","title":"xTrimoPGLM: Unified 100B-Scale Pre-trained Transformer for Deciphering the Language of Protein","created":"2024-01-11","cited_by_count":50,"area":null},{"id":"2112.12731","title":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-12-23","cited_by_count":46,"area":null},{"id":"2312.15159","title":"Understanding the Potential of FPGA-Based Spatial Acceleration for Large Language Model Inference","created":"2023-12-23","cited_by_count":33,"area":null},{"id":"2206.01299","title":"Fine-tuning Language Models over Slow Networks using Activation Compression with Guarantees","created":"2022-06-02","cited_by_count":30,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2211.14133","title":"PipeFisher: Efficient Training of Large Language Models Using Pipelining and Fisher Information Matrices","created":"2022-11-25","cited_by_count":24,"area":null},{"id":"2201.07598","title":"Near-Optimal Sparse Allreduce for Distributed Deep Learning","created":"2022-01-19","cited_by_count":23,"area":null},{"id":"2312.12391","title":"vTrain: A Simulation Framework for Evaluating Cost-effective and Compute-optimal Large Language Model Training","created":"2023-11-27","cited_by_count":23,"area":null},{"id":"2312.06902","title":"Reducing Energy Bloat in Large Model Training","created":"2023-12-12","cited_by_count":22,"area":null},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20,"area":null},{"id":"2211.05322","title":"On Optimizing the Communication of Model Parallelism","created":"2022-11-10","cited_by_count":19,"area":null},{"id":"2412.01523","title":"FlexSP: Accelerating Large Language Model Training via Flexible Sequence Parallelism","created":"2024-12-02","cited_by_count":19,"area":null},{"id":"2209.06800","title":"MGG: Accelerating Graph Neural Networks with Fine-grained intra-kernel Communication-Computation Pipelining on Multi-GPU Platforms","created":"2022-09-14","cited_by_count":16,"area":null},{"id":"2401.00134","title":"Unicron: Economizing Self-Healing LLM Training at Scale","created":"2023-12-30","cited_by_count":16,"area":null},{"id":"2407.13853","title":"Forecasting GPU Performance for Deep Learning Training and Inference","created":"2024-07-18","cited_by_count":15,"area":null},{"id":"2307.02031","title":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","created":"2023-07-05","cited_by_count":14,"area":null},{"id":"2107.01499","title":"BAGUA: Scaling up Distributed Learning with System Relaxations","created":"2021-07-03","cited_by_count":13,"area":null},{"id":"2110.14895","title":"Pipeline Parallelism for Inference on Heterogeneous Edge Computing","created":"2021-10-28","cited_by_count":13,"area":null},{"id":"2212.04129","title":"Deep Incubation: Training Large Models by Divide-and-Conquering","created":"2022-12-08","cited_by_count":13,"area":null},{"id":"2112.02958","title":"Automap: Towards Ergonomic Automated Parallelism for ML Models","created":"2021-12-06","cited_by_count":12,"area":null},{"id":"2301.02654","title":"Does compressing activations help model parallel training?","created":"2023-01-06","cited_by_count":12,"area":null},{"id":"2111.05972","title":"Amazon SageMaker Model Parallelism: A General and Flexible Framework for Large Model Training","created":"2021-11-10","cited_by_count":11,"area":null},{"id":"2206.04959","title":"Merak: An Efficient Distributed DNN Training Framework with Automated 3D Parallelism for Giant Foundation Models","created":"2022-06-10","cited_by_count":11,"area":null},{"id":"2401.02643","title":"Training and Serving System of Foundation Models: A Comprehensive Survey","created":"2024-01-05","cited_by_count":11,"area":null},{"id":"2411.19870","title":"DeMo: Decoupled Momentum Optimization","created":"2024-11-29","cited_by_count":11,"area":null},{"id":"2502.12996","title":"Eager Updates For Overlapped Communication and Computation in DiLoCo","created":"2025-02-18","cited_by_count":11,"area":null},{"id":"2401.11202","title":"PartIR: Composing SPMD Partitioning Strategies for Machine Learning","created":"2024-01-20","cited_by_count":10,"area":null},{"id":"2403.10616","title":"DiPaCo: Distributed Path Composition","created":"2024-03-15","cited_by_count":10,"area":null},{"id":"2406.17145","title":"GraphPipe: Improving Performance and Scalability of DNN Training with Graph Pipeline Parallelism","created":"2024-06-24","cited_by_count":9,"area":null},{"id":"2407.14645","title":"Performance Modeling and Workload Analysis of Distributed Large Language Model Training and Inference","created":"2024-07-19","cited_by_count":9,"area":null},{"id":"2309.15467","title":"Enabling Resource-efficient AIoT System with Cross-level Optimization: A survey","created":"2023-09-27","cited_by_count":8,"area":null},{"id":"2311.05610","title":"Efficient Parallelization Layouts for Large-Scale Distributed Model Training","created":"2023-11-09","cited_by_count":8,"area":null},{"id":"2103.16063","title":"Automatic Graph Partitioning for Very Large-scale Deep Learning","created":"2021-03-30","cited_by_count":7,"area":null},{"id":"2210.04024","title":"Demand Layering for Real-Time DNN Inference with Minimized Memory Usage","created":"2022-10-08","cited_by_count":7,"area":null},{"id":"2409.03365","title":"Spindle: Efficient Distributed Training of Multi-Task Large Models via Wavefront Scheduling","created":"2024-09-05","cited_by_count":7,"area":null},{"id":"2503.19050","title":"Mist: Efficient Distributed Training of Large Language Models via Memory-Parallelism Co-Optimization","created":"2025-03-24","cited_by_count":7,"area":null},{"id":"2106.02679","title":"Layered gradient accumulation and modular pipeline parallelism: fast and efficient training of large language models","created":"2021-06-04","cited_by_count":6,"area":null},{"id":"2202.01306","title":"Harmony: Overcoming the Hurdles of GPU Memory Capacity to Train Massive DNN Models on Commodity Servers","created":"2022-02-02","cited_by_count":6,"area":null},{"id":"2406.02052","title":"PETRA: Parallel End-to-end Training with Reversible Architectures","created":"2024-06-04","cited_by_count":6,"area":null},{"id":"2505.01099","title":"Nesterov Method for Asynchronous Pipeline Parallel Optimization","created":"2025-05-02","cited_by_count":6,"area":null},{"id":"2112.02752","title":"End-to-end Adaptive Distributed Training on PaddlePaddle","created":"2021-12-06","cited_by_count":5,"area":null},{"id":"2407.12117","title":"MEMO: Fine-grained Tensor Management For Ultra-long Context LLM Training","created":"2024-07-16","cited_by_count":5,"area":null},{"id":"2509.10371","title":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","created":"2025-09-12","cited_by_count":5,"area":null},{"id":"2202.10435","title":"Survey on Large Scale Neural Network Training","created":"2022-02-21","cited_by_count":4,"area":null},{"id":"2301.00391","title":"PiPAD: Pipelined and Parallel Dynamic GNN Training on GPUs","created":"2023-01-01","cited_by_count":4,"area":null},{"id":"2301.02691","title":"Systems for Parallel and Distributed Large-Model Deep Learning Training","created":"2023-01-06","cited_by_count":4,"area":null},{"id":"2308.10087","title":"GNNPipe: Scaling Deep GNN Training with Pipelined Model Parallelism","created":"2023-08-19","cited_by_count":4,"area":null},{"id":"2312.00839","title":"PipeOptim: Ensuring Effective 1F1B Schedule with Optimizer-Dependent Weight Prediction","created":"2023-12-01","cited_by_count":4,"area":null},{"id":"2312.03549","title":"Holmes: Towards Distributed Training Across Clusters with Heterogeneous NIC Environment","created":"2023-12-06","cited_by_count":4,"area":null},{"id":"2407.13143","title":"Integrated Hardware Architecture and Device Placement Search","created":"2024-07-18","cited_by_count":4,"area":null},{"id":"2503.12053","title":"Ferret: An Efficient Online Continual Learning Framework under Varying Memory Constraints","created":"2025-03-15","cited_by_count":4,"area":null},{"id":"2506.01352","title":"TAH-QUANT: Effective Activation Quantization in Pipeline Parallelism over Slow Network","created":"2025-06-02","cited_by_count":4,"area":null},{"id":"2110.08633","title":"Hydra: A System for Large Multi-Model Deep Learning","created":"2021-10-16","cited_by_count":3,"area":null},{"id":"2205.09470","title":"Nebula-I: A General Framework for Collaboratively Training Deep Learning Models on Low-Bandwidth Cloud Clusters","created":"2022-05-19","cited_by_count":3,"area":null},{"id":"2401.02088","title":"Re-evaluating the Memory-balanced Pipeline Parallelism: BPipe","created":"2024-01-04","cited_by_count":3,"area":null},{"id":"2410.19367","title":"BitPipe: Bidirectional Interleaved Pipeline Parallelism for Accelerating Large Models Training","created":"2024-10-25","cited_by_count":3,"area":null},{"id":"2412.07894","title":"Hydraulis: Balancing Large Transformer Model Training via Co-designing Parallel Strategies and Data Assignment","created":"2024-12-10","cited_by_count":3,"area":null},{"id":"2601.22442","title":"AsyncMesh: Fully Asynchronous Optimization for Data and Pipeline Parallelism","created":"2026-01-30","cited_by_count":3,"area":null},{"id":"2111.05426","title":"DistIR: An Intermediate Representation and Simulator for Efficient Neural Network Distribution","created":"2021-11-09","cited_by_count":2,"area":null},{"id":"2112.05387","title":"Layer-Parallel Training of Residual Networks with Auxiliary-Variable Networks","created":"2021-12-10","cited_by_count":2,"area":null},{"id":"2307.16375","title":"UniAP: Unifying Inter- and Intra-Layer Automatic Parallelism by Mixed Integer Quadratic Programming","created":"2023-07-31","cited_by_count":2,"area":null},{"id":"2401.01728","title":"Ravnest: Decentralized Asynchronous Training on Heterogeneous Devices","created":"2024-01-03","cited_by_count":2,"area":null},{"id":"2405.16256","title":"HETHUB: A Distributed Training System with Heterogeneous Cluster for Large-Scale Models","created":"2024-05-25","cited_by_count":2,"area":null},{"id":"2406.08756","title":"Optimizing Large Model Training through Overlapped Activation Recomputation","created":"2024-06-13","cited_by_count":2,"area":null},{"id":"2411.06465","title":"Accelerating Large Language Model Training with 4D Parallelism and Memory Consumption Estimator","created":"2024-11-10","cited_by_count":2,"area":null},{"id":"2502.16892","title":"Applying LLMs to Active Learning: Towards Cost-Efficient Cross-Task Text Classification without Manually Labeled Data","created":"2025-02-24","cited_by_count":2,"area":null},{"id":"2507.00394","title":"HelixPipe: Efficient Distributed Training of Long Sequence Transformers with Attention Parallel Pipeline Parallelism","created":"2025-07-01","cited_by_count":2,"area":null},{"id":"2602.03515","title":"Mitigating Staleness in Asynchronous Pipeline Parallelism via Basis Rotation","created":"2026-02-03","cited_by_count":2,"area":null},{"id":"2207.11912","title":"Dive into Big Model Training","created":"2022-07-25","cited_by_count":1,"area":null},{"id":"2210.09147","title":"PARTIME: Scalable and Parallel Processing Over Time with Deep Neural Networks","created":"2022-10-17","cited_by_count":1,"area":null},{"id":"2312.04025","title":"Moirai: Towards Optimal Placement for Distributed Inference on Heterogeneous Devices","created":"2023-12-07","cited_by_count":1,"area":null},{"id":"2602.21788","title":"Efficient Scaling of LLM Training with Flexible Context Parallelism","created":"2026-02-25","cited_by_count":1,"area":null},{"id":"2605.29664","title":"AMDP: Asynchronous Multi-Directional Pipeline Parallelism for Large-Scale Models Training","created":"2026-05-28","cited_by_count":1,"area":null},{"id":"2110.00929","title":"Scheduling Optimization Techniques for Neural Network Training","created":"2021-10-03","cited_by_count":0,"area":null},{"id":"2202.11575","title":"Shisha: Online scheduling of CNN pipelines on heterogeneous architectures","created":"2022-02-23","cited_by_count":0,"area":null},{"id":"2301.13799","title":"Partitioning Distributed Compute Jobs with Reinforcement Learning and Graph Neural Networks","created":"2023-01-31","cited_by_count":0,"area":null},{"id":"2303.05330","title":"Cloudless-Training: A Framework to Improve Efficiency of Geo-Distributed ML Training","created":"2023-03-09","cited_by_count":0,"area":null},{"id":"2311.03703","title":"Practical Performance Guarantees for Pipelined DNN Inference","created":"2023-11-07","cited_by_count":0,"area":null},{"id":"2311.05827","title":"AccEPT: An Acceleration Scheme for Speeding Up Edge Pipeline-parallel Training","created":"2023-11-10","cited_by_count":0,"area":null},{"id":"2403.08837","title":"Cyclic Data Parallelism for Efficient Parallelism of Deep Neural Networks","created":"2024-03-13","cited_by_count":0,"area":null},{"id":"2409.06941","title":"FreeRide: Harvesting Bubbles in Pipeline Parallelism","created":"2024-09-11","cited_by_count":0,"area":null},{"id":"2501.05563","title":"Prediction-Assisted Online Distributed Deep Learning Workload Scheduling in GPU Clusters","created":"2025-01-09","cited_by_count":0,"area":null},{"id":"2502.02893","title":"Lowering the Barrier of Machine Learning: Achieving Zero Manual Labeling in Review Classification Using LLMs","created":"2025-02-05","cited_by_count":0,"area":null},{"id":"2503.09357","title":"Automatic Operator-level Parallelism Planning for Distributed Deep Learning -- A Mixed-Integer Programming Approach","created":"2025-03-12","cited_by_count":0,"area":null},{"id":"2503.15758","title":"ATTENTION2D: Communication Efficient Distributed Self-Attention Mechanism","created":"2025-03-20","cited_by_count":0,"area":null},{"id":"2504.21411","title":"Galvatron: An Automatic Distributed System for Efficient Foundation Model Training","created":"2025-04-30","cited_by_count":0,"area":null}],"next_cursor":"100"}