{"total":117,"rows":[{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073,"area":"moe"},{"id":"2303.18223","title":"A Survey of Large Language Models","created":"2023-03-31","cited_by_count":2576,"area":null},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631,"area":"moe"},{"id":"2401.06066","title":"DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models","created":"2024-01-11","cited_by_count":867,"area":"moe"},{"id":"2412.19437","title":"DeepSeek-V3 Technical Report","created":"2024-12-27","cited_by_count":811,"area":"lr"},{"id":"2204.06745","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","created":"2022-04-14","cited_by_count":713,"area":null},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660,"area":null},{"id":"2507.20534","title":"Kimi K2: Open Agentic Intelligence","created":"2025-07-28","cited_by_count":567,"area":"agents"},{"id":"2207.00032","title":"DeepSpeed Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale","created":"2022-06-30","cited_by_count":407,"area":null},{"id":"2110.02861","title":"8-bit Optimizers via Block-wise Quantization","created":"2021-10-06","cited_by_count":403,"area":null},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2310.05694","title":"A Survey of Large Language Models for Healthcare: from Data, Technology, and Applications to Accountability and Ethics","created":"2023-10-09","cited_by_count":163,"area":null},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2104.05158","title":"Software-Hardware Co-design for Fast and Scalable Training of Deep Learning Recommendation Models","created":"2021-04-12","cited_by_count":78,"area":null},{"id":"2109.10465","title":"Scalable and Efficient MoE Training for Multitask Multilingual Models","created":"2021-09-22","cited_by_count":76,"area":"moe"},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75,"area":null},{"id":"2311.12351","title":"Advancing Transformer Architecture in Long-Context Large Language Models: A Comprehensive Survey","created":"2023-11-21","cited_by_count":72,"area":null},{"id":"2512.16776","title":"Kling-Omni Technical Report","created":"2025-12-18","cited_by_count":71,"area":null},{"id":"2501.08453","title":"Vchitect-2.0: Parallel Transformer for Scaling Up Video Diffusion Models","created":"2025-01-14","cited_by_count":60,"area":"diffusion"},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52,"area":null},{"id":"2303.14006","title":"ASTRA-sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-model Training at Scale","created":"2023-03-24","cited_by_count":47,"area":null},{"id":"2112.12731","title":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-12-23","cited_by_count":46,"area":null},{"id":"2203.10428","title":"PipeGCN: Efficient Full-Graph Training of Graph Convolutional Networks with Pipelined Feature Communication","created":"2022-03-20","cited_by_count":45,"area":null},{"id":"2203.10983","title":"BNS-GCN: Efficient Full-Graph Training of Graph Convolutional Networks with Partition-Parallelism and Random Boundary Node Sampling","created":"2022-03-21","cited_by_count":43,"area":null},{"id":"2401.10241","title":"Zero Bubble Pipeline Parallelism","created":"2023-11-30","cited_by_count":40,"area":null},{"id":"2308.12219","title":"Diffusion Language Models Can Perform Many Tasks with Scaling and Instruction-Finetuning","created":"2023-08-23","cited_by_count":39,"area":"diffusion"},{"id":"2406.14088","title":"ReaL: Efficient RLHF Training of Large Language Models with Parameter Reallocation","created":"2024-06-20","cited_by_count":37,"area":"alignment"},{"id":"2410.05261","title":"TextHawk2: A Large Vision-Language Model Excels in Bilingual OCR and Grounding with 16x Fewer Tokens","created":"2024-10-07","cited_by_count":32,"area":"mllm"},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2105.14450","title":"Maximizing Parallelism in Distributed Training for Huge Neural Networks","created":"2021-05-30","cited_by_count":23,"area":null},{"id":"2410.07265","title":"A Survey: Collaborative Hardware and Software Design in the Era of Large Language Models","created":"2024-10-08","cited_by_count":23,"area":null},{"id":"2410.11305","title":"QSpec: Speculative Decoding with Complementary Quantization Schemes","created":"2024-10-15","cited_by_count":18,"area":"inference"},{"id":"2111.05897","title":"Persia: An Open, Hybrid System Scaling Deep Learning-based Recommenders up to 100 Trillion Parameters","created":"2021-11-10","cited_by_count":17,"area":null},{"id":"2110.04478","title":"Themis: A Network Bandwidth-Aware Collective Scheduling Policy for Distributed Training of DL Models","created":"2021-10-09","cited_by_count":16,"area":null},{"id":"2504.19720","title":"Taming the Titans: A Survey of Efficient LLM Inference Serving","created":"2025-04-28","cited_by_count":16,"area":"inference"},{"id":"2105.01281","title":"Citadel: Protecting Data Privacy and Model Confidentiality for Collaborative Learning with SGX","created":"2021-05-04","cited_by_count":14,"area":null},{"id":"2302.02390","title":"Quantized Distributed Training of Large Models with Convergence Guarantees","created":"2023-02-05","cited_by_count":13,"area":null},{"id":"2312.11819","title":"An Adaptive Placement and Parallelism Framework for Accelerating RLHF Training","created":"2023-12-19","cited_by_count":13,"area":"alignment"},{"id":"2405.15362","title":"Pipeline Parallelism with Controllable Memory","created":"2024-05-24","cited_by_count":11,"area":null},{"id":"2409.15241","title":"Domino: Eliminating Communication in LLM Training via Generic Tensor Slicing and Overlapping","created":"2024-09-23","cited_by_count":9,"area":null},{"id":"2112.01637","title":"AdaSplit: Adaptive Trade-offs for Resource-constrained Distributed Deep Learning","created":"2021-12-02","cited_by_count":8,"area":null},{"id":"2209.01346","title":"HammingMesh: A Network Topology for Large-Scale Deep Learning","created":"2022-09-03","cited_by_count":8,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2103.16063","title":"Automatic Graph Partitioning for Very Large-scale Deep Learning","created":"2021-03-30","cited_by_count":7,"area":null},{"id":"2307.07950","title":"Accelerating Distributed ML Training via Selective Synchronization","created":"2023-07-16","cited_by_count":7,"area":null},{"id":"2401.07788","title":"Activations and Gradients Compression for Model-Parallel Training","created":"2024-01-15","cited_by_count":7,"area":null},{"id":"2402.17812","title":"DropBP: Accelerating Fine-Tuning of Large Language Models by Dropping Backward Propagation","created":"2024-02-27","cited_by_count":7,"area":null},{"id":"2403.03699","title":"Model Parallelism on Distributed Infrastructure: A Literature Review from Theory to LLM Case-Studies","created":"2024-03-06","cited_by_count":7,"area":null},{"id":"2408.10197","title":"Demystifying the Communication Characteristics for Distributed Transformer Models","created":"2024-08-19","cited_by_count":7,"area":null},{"id":"2106.03640","title":"Making EfficientNet More Efficient: Exploring Batch-Independent Normalization, Group Convolutions and Reduced Resolution Training","created":"2021-06-07","cited_by_count":6,"area":null},{"id":"2406.02052","title":"PETRA: Parallel End-to-end Training with Reversible Architectures","created":"2024-06-04","cited_by_count":6,"area":null},{"id":"2407.04480","title":"LoCo: Low-Bit Communication Adaptor for Large-scale Model Training","created":"2024-07-05","cited_by_count":6,"area":null},{"id":"2503.01328","title":"PipeOffload: Improving Scalability of Pipeline Parallelism with Memory Optimization","created":"2025-03-03","cited_by_count":5,"area":null},{"id":"2504.14775","title":"gLLM: Global Balanced Pipeline Parallelism System for Distributed LLM Serving with Token Throttling","created":"2025-04-21","cited_by_count":5,"area":"inference"},{"id":"2509.05276","title":"SpikingBrain: Spiking Brain-inspired Large Models","created":"2025-09-05","cited_by_count":5,"area":null},{"id":"2509.10371","title":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","created":"2025-09-12","cited_by_count":5,"area":null},{"id":"2301.02691","title":"Systems for Parallel and Distributed Large-Model Deep Learning Training","created":"2023-01-06","cited_by_count":4,"area":null},{"id":"2304.11414","title":"Pipeline MoE: A Flexible MoE Implementation with Pipeline Parallelism","created":"2023-04-22","cited_by_count":4,"area":"moe"},{"id":"2502.19913","title":"SkipPipe: Partial and Reordered Pipelining Framework for Training LLMs in Heterogeneous Networks","created":"2025-02-27","cited_by_count":4,"area":null},{"id":"2605.17862","title":"$\\boldsymbol{f}$-OPD: Stabilizing Long-Horizon On-Policy Distillation with Freshness-Aware Control","created":"2026-05-18","cited_by_count":4,"area":"distill"},{"id":"2110.08633","title":"Hydra: A System for Large Multi-Model Deep Learning","created":"2021-10-16","cited_by_count":3,"area":null},{"id":"2205.09470","title":"Nebula-I: A General Framework for Collaboratively Training Deep Learning Models on Low-Bandwidth Cloud Clusters","created":"2022-05-19","cited_by_count":3,"area":null},{"id":"2211.05953","title":"Breadth-First Pipeline Parallelism","created":"2022-11-11","cited_by_count":3,"area":null},{"id":"2308.08500","title":"InTune: Reinforcement Learning-based Data Pipeline Optimization for Deep Recommendation Models","created":"2023-08-13","cited_by_count":3,"area":null},{"id":"2412.07894","title":"Hydraulis: Balancing Large Transformer Model Training via Co-designing Parallel Strategies and Data Assignment","created":"2024-12-10","cited_by_count":3,"area":null},{"id":"2506.05871","title":"BestServe: Serving Strategies with Optimal Goodput in Collocation and Disaggregation Architectures","created":"2025-06-06","cited_by_count":3,"area":null},{"id":"2511.04951","title":"CLM: Removing the GPU Memory Barrier for 3D Gaussian Splatting","created":"2025-11-07","cited_by_count":3,"area":null},{"id":"2606.01725","title":"Characterization of Multi-Model Agentic AI Systems on General Tasks via Trace-Driven Simulation","created":"2026-06-01","cited_by_count":3,"area":"agents"},{"id":"2106.05822","title":"GroupBERT: Enhanced Transformer Architecture with Efficient Grouped Structures","created":"2021-06-10","cited_by_count":2,"area":null},{"id":"2112.05387","title":"Layer-Parallel Training of Residual Networks with Auxiliary-Variable Networks","created":"2021-12-10","cited_by_count":2,"area":null},{"id":"2203.15980","title":"DELTA: Dynamically Optimizing GPU Memory beyond Tensor Recomputation","created":"2022-03-30","cited_by_count":2,"area":null},{"id":"2301.08695","title":"Baechi: Fast Device Placement of Machine Learning Graphs","created":"2023-01-20","cited_by_count":2,"area":null},{"id":"2302.11180","title":"DISCO: Distributed Inference with Sparse Communications","created":"2023-02-22","cited_by_count":2,"area":null},{"id":"2309.01226","title":"Saturn: An Optimized Data System for Large Model Deep Learning Workloads","created":"2023-09-03","cited_by_count":2,"area":null},{"id":"2401.01728","title":"Ravnest: Decentralized Asynchronous Training on Heterogeneous Devices","created":"2024-01-03","cited_by_count":2,"area":null},{"id":"2405.16256","title":"HETHUB: A Distributed Training System with Heterogeneous Cluster for Large-Scale Models","created":"2024-05-25","cited_by_count":2,"area":null},{"id":"2412.12371","title":"Priority-Aware Model-Distributed Inference at Edge Networks","created":"2024-12-16","cited_by_count":2,"area":null},{"id":"2508.07970","title":"WeChat-YATT: A Scalable, Simple, Efficient, and Production Ready Training Library","created":"2025-08-11","cited_by_count":2,"area":"alignment"},{"id":"2512.20968","title":"Mesh-Attention: A New Communication-Efficient Distributed Attention with Improved Data Locality","created":"2025-12-24","cited_by_count":2,"area":null},{"id":"2204.10943","title":"FPGA-based AI Smart NICs for Scalable Distributed AI Training Systems","created":"2022-04-22","cited_by_count":1,"area":null},{"id":"2207.11912","title":"Dive into Big Model Training","created":"2022-07-25","cited_by_count":1,"area":null},{"id":"2404.06709","title":"CQIL: Inference Latency Optimization with Concurrent Computation of Quasi-Independent Layers","created":"2024-04-10","cited_by_count":1,"area":null},{"id":"2506.22175","title":"MPipeMoE: Memory Efficient MoE for Pre-trained Models with Adaptive Pipeline Parallelism","created":"2025-06-27","cited_by_count":1,"area":"moe"},{"id":"2507.22789","title":"G-Core: A Simple, Scalable and Balanced RLHF Trainer","created":"2025-07-30","cited_by_count":1,"area":"diffusion"},{"id":"2511.10480","title":"Scalable Synthesis of distributed LLM workloads through Symbolic Tensor Graphs","created":"2025-11-13","cited_by_count":1,"area":null},{"id":"2603.05692","title":"Parallelization Strategies for Dense LLM Deployment: Navigating Through Application-Specific Tradeoffs and Bottlenecks","created":"2026-03-05","cited_by_count":1,"area":null},{"id":"2608.03487","title":"RAG-Stack: Co-Optimizing RAG Serving Performance and Quality","created":"2026-08-04","cited_by_count":1,"area":"rag"},{"id":"2610.05305","title":"Characterizing Parallelism Strategies in LLM Inference: Fundamental Compute-Communication Trade-offs","created":"2026-10-04","cited_by_count":1,"area":null},{"id":"2104.05343","title":"An Efficient 2D Method for Training Super-Large Deep Learning Models","created":"2021-04-12","cited_by_count":0,"area":null},{"id":"2110.02781","title":"FTPipeHD: A Fault-Tolerant Pipeline-Parallel Distributed Training Framework for Heterogeneous Edge Devices","created":"2021-10-06","cited_by_count":0,"area":null},{"id":"2310.15334","title":"ADMM Algorithms for Residual Network Training: Convergence Analysis and Parallel Implementation","created":"2023-10-23","cited_by_count":0,"area":null},{"id":"2311.05827","title":"AccEPT: An Acceleration Scheme for Speeding Up Edge Pipeline-parallel Training","created":"2023-11-10","cited_by_count":0,"area":null},{"id":"2401.11471","title":"LR-CNN: Lightweight Row-centric Convolutional Neural Network Training for Memory Reduction","created":"2024-01-21","cited_by_count":0,"area":null},{"id":"2402.11427","title":"OptEx: Expediting First-Order Optimization with Approximately Parallelized Iterations","created":"2024-02-18","cited_by_count":0,"area":null},{"id":"2405.18047","title":"2BP: 2-Stage Backpropagation","created":"2024-05-28","cited_by_count":0,"area":null},{"id":"2407.00467","title":"VcLLM: Video Codecs are Secretly Tensor Codecs","created":"2024-06-29","cited_by_count":0,"area":null},{"id":"2501.01951","title":"MixGCN: Scalable GCN Training by Mixture of Parallelism and Mixture of Accelerators","created":"2025-01-03","cited_by_count":0,"area":null},{"id":"2504.06095","title":"Nonuniform-Tensor-Parallelism: Mitigating GPU failure impact for Scaled-up LLM Training","created":"2025-04-08","cited_by_count":0,"area":null},{"id":"2505.14864","title":"Balanced and Elastic End-to-end Training of Dynamic LLMs","created":"2025-05-20","cited_by_count":0,"area":"inference"}],"next_cursor":"100"}