{"list":"other","total":92,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"1907.11692","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","created":"2019-07-26","cited_by_count":13372},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1910.13461","title":"BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension","created":"2019-10-29","cited_by_count":5893},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2403.05530","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","created":"2024-03-08","cited_by_count":3224},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"2303.18223","title":"A Survey of Large Language Models","created":"2023-03-31","cited_by_count":2576},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","created":"2020-06-29","cited_by_count":2022},{"id":"1808.06226","title":"SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing","created":"2018-08-19","cited_by_count":1989},{"id":"2211.05100","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","created":"2022-11-09","cited_by_count":1984},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"2409.12186","title":"Qwen2.5-Coder Technical Report","created":"2024-09-18","cited_by_count":1203},{"id":"2406.17557","title":"The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale","created":"2024-06-25","cited_by_count":1160},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897},{"id":"1911.02150","title":"Fast Transformer Decoding: One Write-Head is All You Need","created":"2019-11-06","cited_by_count":717},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"2402.00159","title":"Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research","created":"2024-01-31","cited_by_count":483},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2406.11794","title":"DataComp-LM: In search of the next generation of training sets for language models","created":"2024-06-17","cited_by_count":398},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2309.16039","title":"Effective Long-Context Scaling of Foundation Models","created":"2023-09-27","cited_by_count":303},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245},{"id":"2304.01433","title":"TPU v4: An Optically Reconfigurable Supercomputer for Machine Learning with Hardware Support for Embeddings","created":"2023-04-04","cited_by_count":202},{"id":"1807.05358","title":"Beyond Data and Model Parallelism for Deep Neural Networks","created":"2018-07-14","cited_by_count":179},{"id":"1811.02084","title":"Mesh-TensorFlow: Deep Learning for Supercomputers","created":"2018-11-05","cited_by_count":169},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126},{"id":"1806.03377","title":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","created":"2018-06-08","cited_by_count":117},{"id":"2402.15627","title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs","created":"2024-02-23","cited_by_count":112},{"id":"2105.04663","title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","created":"2021-05-10","cited_by_count":111},{"id":"2210.17223","title":"Accelerating Distributed MoE Training and Inference with Lina","created":"2022-10-31","cited_by_count":109},{"id":"2310.08754","title":"Tokenizer Choice For LLM Training: Negligible or Crucial?","created":"2023-10-12","cited_by_count":108},{"id":"2301.10472","title":"XLM-V: Overcoming the Vocabulary Bottleneck in Multilingual Masked Language Models","created":"2023-01-25","cited_by_count":88},{"id":"2012.15524","title":"Fast WordPiece Tokenization","created":"2020-12-31","cited_by_count":87},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75},{"id":"2410.21676","title":"How Does Critical Batch Size Scale in Pre-training?","created":"2024-10-29","cited_by_count":73},{"id":"2402.18041","title":"Datasets for Large Language Models: A Comprehensive Survey","created":"2024-02-28","cited_by_count":71},{"id":"2405.07719","title":"USP: A Unified Sequence Parallelism Approach for Long Context Generative AI","created":"2024-05-13","cited_by_count":61},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53},{"id":"2303.14006","title":"ASTRA-sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-model Training at Scale","created":"2023-03-24","cited_by_count":47},{"id":"1903.04611","title":"Evaluating Modern GPU Interconnect: PCIe, NVLink, NV-SLI, NVSwitch and GPUDirect","created":"2019-03-11","cited_by_count":41},{"id":"2008.08708","title":"Synthesizing Optimal Collective Algorithms","created":"2020-08-19","cited_by_count":30},{"id":"2402.06853","title":"History, Development, and Principles of Large Language Models-An Introductory Survey","created":"2024-02-10","cited_by_count":30},{"id":"2310.03294","title":"DISTFLASHATTN: Distributed Memory-efficient Attention for Long-context LLMs Training","created":"2023-10-05","cited_by_count":29},{"id":"2604.21691","title":"There Will Be a Scientific Theory of Deep Learning","created":"2026-04-23","cited_by_count":29},{"id":"2407.20018","title":"Efficient Training of Large Language Models on Distributed Infrastructures: A Survey","created":"2024-07-29","cited_by_count":27},{"id":"2407.20143","title":"ByteCheckpoint: A Unified Checkpointing System for Large Foundation Model Development","created":"2024-07-29","cited_by_count":24},{"id":"2507.19595","title":"Efficient Attention Mechanisms for Large Language Models: A Survey","created":"2025-07-25","cited_by_count":22},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20},{"id":"2507.04786","title":"Demystifying NCCL: An In-depth Analysis of GPU Communication Protocols and Algorithms","created":"2025-07-07","cited_by_count":20},{"id":"2501.10714","title":"FSMoE: A Flexible and Scalable Training System for Sparse Mixture-of-Experts Models","created":"2025-01-18","cited_by_count":17},{"id":"2503.17924","title":"WLB-LLM: Workload-Balanced 4D Parallelism for Large Language Model Training","created":"2025-03-23","cited_by_count":17},{"id":"2307.12169","title":"Rail-only: A Low-Cost High-Performance Network for Training LLMs with Trillion Parameters","created":"2023-07-22","cited_by_count":13},{"id":"2505.05713","title":"Understanding Stragglers in Large Model Training Using What-if Analysis","created":"2025-05-09","cited_by_count":12},{"id":"1910.05930","title":"Characterizing Deep Learning Training Workloads on Alibaba-PAI","created":"2019-10-14","cited_by_count":11},{"id":"2305.13479","title":"Rethinking Machine Learning Collective Communication as a Multi-Commodity Flow Problem","created":"2023-05-22","cited_by_count":11},{"id":"2405.15362","title":"Pipeline Parallelism with Controllable Memory","created":"2024-05-24","cited_by_count":11},{"id":"2404.06114","title":"Communication-Efficient Large-Scale Distributed Deep Learning: A Comprehensive Survey","created":"2024-04-09","cited_by_count":10},{"id":"2504.09014","title":"MSCCL++: Rethinking GPU Communication Abstractions for AI Inference","created":"2025-04-11","cited_by_count":10},{"id":"1806.08159","title":"Revisiting Network Support for RDMA","created":"2018-06-21","cited_by_count":8},{"id":"2310.06993","title":"OptiReduce: Resilient and Tail-Optimal AllReduce for Distributed Deep Learning in the Cloud","created":"2023-10-10","cited_by_count":8},{"id":"2408.10197","title":"Demystifying the Communication Characteristics for Distributed Transformer Models","created":"2024-08-19","cited_by_count":7},{"id":"2502.17356","title":"Random Scaling of Emergent Capabilities","created":"2025-02-24","cited_by_count":7},{"id":"2504.09307","title":"Lumos: Efficient Performance Modeling and Estimation for Large-scale LLM Training","created":"2025-04-12","cited_by_count":7},{"id":"2305.13525","title":"A 4D Hybrid Algorithm to Scale Parallel Training to Thousands of GPUs","created":"2023-05-22","cited_by_count":6},{"id":"2411.05288","title":"Balancing Pipeline Parallelism with Vocabulary Parallelism","created":"2024-11-08","cited_by_count":6},{"id":"2509.15940","title":"Efficient Pre-Training of LLMs via Topology-Aware Communication Alignment on More Than 9600 GPUs","created":"2025-09-19","cited_by_count":6},{"id":"2601.02907","title":"Beyond the Black Box: A Survey on the Theory and Mechanism of Large Language Models","created":"2026-01-06","cited_by_count":6},{"id":"2509.10371","title":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","created":"2025-09-12","cited_by_count":5},{"id":"2409.04833","title":"Achieving Peak Performance for Large Language Models: A Systematic Review","created":"2024-09-07","cited_by_count":4},{"id":"2508.00331","title":"Embryology of a Language Model","created":"2025-08-01","cited_by_count":4},{"id":"2403.07585","title":"Communication Optimization for Distributed Training: Architecture, Advances, and Opportunities","created":"2024-03-12","cited_by_count":3},{"id":"2506.15006","title":"Scaling Intelligence: Designing Data Centers for Next-Gen Language Models","created":"2025-06-17","cited_by_count":3},{"id":"2405.09499","title":"A Comprehensive Survey on SmartNICs: Architectures, Development Models, Applications, and Research Directions","created":"2024-05-15","cited_by_count":2},{"id":"2401.16736","title":"Engineering A Large Language Model From Scratch","created":"2024-01-30","cited_by_count":1},{"id":"2608.15118","title":"Collective Communication for Distributed LLM Systems: Planning, Runtime Adaptation, and Computation Coordination","created":"2026-08-15","cited_by_count":1}],"next_cursor":null}