{"list":"other","total":70,"rows":[{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2505.09388","title":"Qwen3 Technical Report","created":"2025-05-14","cited_by_count":8820},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2501.12948","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","created":"2025-01-22","cited_by_count":6330},{"id":"2412.15115","title":"Qwen2.5 Technical Report","created":"2024-12-19","cited_by_count":4942},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2004.05150","title":"Longformer: The Long-Document Transformer","created":"2020-04-10","cited_by_count":3413},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"2409.19256","title":"HybridFlow: A Flexible and Efficient RLHF Framework","created":"2024-09-28","cited_by_count":2057},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2401.04088","title":"Mixtral of Experts","created":"2024-01-08","cited_by_count":1823},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"2007.14062","title":"Big Bird: Transformers for Longer Sequences","created":"2020-07-28","cited_by_count":1684},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"2405.04434","title":"DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model","created":"2024-05-07","cited_by_count":1135},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897},{"id":"2401.06066","title":"DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models","created":"2024-01-11","cited_by_count":867},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"2412.19437","title":"DeepSeek-V3 Technical Report","created":"2024-12-27","cited_by_count":811},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2202.09368","title":"Mixture-of-Experts with Expert Choice Routing","created":"2022-02-18","cited_by_count":623},{"id":"2407.08608","title":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","created":"2024-07-11","cited_by_count":519},{"id":"2106.08295","title":"A White Paper on Neural Network Quantization","created":"2021-06-15","cited_by_count":472},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2110.02861","title":"8-bit Optimizers via Block-wise Quantization","created":"2021-10-06","cited_by_count":403},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399},{"id":"2404.19737","title":"Better & Faster Large Language Models via Multi-token Prediction","created":"2024-04-30","cited_by_count":360},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302},{"id":"1312.4314","title":"Learning Factored Representations in a Deep Mixture of Experts","created":"2013-12-16","cited_by_count":278},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245},{"id":"2212.05055","title":"Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints","created":"2022-12-09","cited_by_count":209},{"id":"2408.15664","title":"Auxiliary-Loss-Free Load Balancing Strategy for Mixture-of-Experts","created":"2024-08-28","cited_by_count":205},{"id":"2209.05433","title":"FP8 Formats for Deep Learning","created":"2022-09-12","cited_by_count":196},{"id":"1811.03600","title":"Measuring the Effects of Data Parallelism on Neural Network Training","created":"2018-11-08","cited_by_count":193},{"id":"2402.07871","title":"Scaling Laws for Fine-Grained Mixture of Experts","created":"2024-02-12","cited_by_count":173},{"id":"2310.10537","title":"Microscaling Data Formats for Deep Learning","created":"2023-10-16","cited_by_count":152},{"id":"2208.09225","title":"FP8 Quantization: The Power of the Exponent","created":"2022-08-19","cited_by_count":88},{"id":"2310.18313","title":"FP8-LM: Training FP8 Large Language Models","created":"2023-10-27","cited_by_count":77},{"id":"2509.25149","title":"Pretraining Large Language Models with NVFP4","created":"2025-09-29","cited_by_count":76},{"id":"2510.11370","title":"Stabilizing MoE Reinforcement Learning by Aligning Training and Inference Routers","created":"2025-10-13","cited_by_count":72},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68},{"id":"2409.12517","title":"Scaling FP8 training to trillion-token LLMs","created":"2024-09-19","cited_by_count":65},{"id":"2512.20856","title":"NVIDIA Nemotron 3: Efficient and Open Intelligence","created":"2025-12-24","cited_by_count":65},{"id":"2406.06858","title":"FLUX: Fast Software-based Communication Overlap On GPUs Through Kernel Fusion","created":"2024-06-11","cited_by_count":57},{"id":"2311.09431","title":"Striped Attention: Faster Ring Attention for Causal Transformers","created":"2023-11-15","cited_by_count":54},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52},{"id":"2410.07524","title":"Upcycling Large Language Models into Mixture of Experts","created":"2024-10-10","cited_by_count":49},{"id":"2401.10241","title":"Zero Bubble Pipeline Parallelism","created":"2023-11-30","cited_by_count":40},{"id":"2510.22115","title":"Every Activation Boosted: Scaling General Reasoner to 1 Trillion Open Language Foundation","created":"2025-10-25","cited_by_count":34},{"id":"2410.19313","title":"COAT: Compressing Optimizer states and Activation for Memory-Efficient FP8 Training","created":"2024-10-25","cited_by_count":33},{"id":"2502.05172","title":"Joint MoE Scaling Laws: Mixture of Experts Can Be Memory Efficient","created":"2025-02-07","cited_by_count":31},{"id":"1911.13214","title":"Optimal checkpointing for heterogeneous chains: how to train deep neural networks with limited memory","created":"2019-11-27","cited_by_count":22},{"id":"2503.17924","title":"WLB-LLM: Workload-Balanced 4D Parallelism for Large Language Model Training","created":"2025-03-23","cited_by_count":17},{"id":"2405.15362","title":"Pipeline Parallelism with Controllable Memory","created":"2024-05-24","cited_by_count":11},{"id":"1811.06965","title":"GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism","created":"2018-11-16","cited_by_count":10},{"id":"2502.21231","title":"ByteScale: Efficient Scaling of LLM Training with a 2048K Context Length on More Than 12,000 GPUs","created":"2025-02-28","cited_by_count":10},{"id":"2107.02027","title":"Efficient Sequence Packing without Cross-contamination: Accelerating Large Language Models without Impacting Performance","created":"2021-06-29","cited_by_count":8},{"id":"2412.09952","title":"Llama 3 Meets MoE: Efficient Upcycling","created":"2024-12-13","cited_by_count":6},{"id":"2206.06356","title":"Modern Distributed Data-Parallel Large-Scale Pre-training Strategies For NLP models","created":"2022-06-13","cited_by_count":5},{"id":"2312.00839","title":"PipeOptim: Ensuring Effective 1F1B Schedule with Optimizer-Dependent Weight Prediction","created":"2023-12-01","cited_by_count":4},{"id":"2308.05548","title":"Learning (With) Distributed Optimization","created":"2023-08-10","cited_by_count":1}],"next_cursor":null}