{"list":"other","total":84,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2505.09388","title":"Qwen3 Technical Report","created":"2025-05-14","cited_by_count":8820},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1608.03983","title":"SGDR: Stochastic Gradient Descent with Warm Restarts","created":"2016-08-13","cited_by_count":5045},{"id":"2412.15115","title":"Qwen2.5 Technical Report","created":"2024-12-19","cited_by_count":4942},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"1907.10902","title":"Optuna: A Next-generation Hyperparameter Optimization Framework","created":"2019-07-25","cited_by_count":2501},{"id":"2408.00118","title":"Gemma 2: Improving Open Language Models at a Practical Size","created":"2024-07-31","cited_by_count":1998},{"id":"2503.19786","title":"Gemma 3 Technical Report","created":"2025-03-25","cited_by_count":1824},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"1606.04838","title":"Optimization Methods for Large-Scale Machine Learning","created":"2016-06-15","cited_by_count":1150},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103},{"id":"1804.04235","title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost","created":"2018-04-11","cited_by_count":905},{"id":"2404.06395","title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","created":"2024-04-09","cited_by_count":723},{"id":"1603.06560","title":"Hyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization","created":"2016-03-21","cited_by_count":699},{"id":"1603.04467","title":"TensorFlow: Large-Scale Machine Learning on Heterogeneous Distributed Systems","created":"2016-03-14","cited_by_count":664},{"id":"2401.02954","title":"DeepSeek LLM: Scaling Open-Source Language Models with Longtermism","created":"2024-01-05","cited_by_count":642},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535},{"id":"2010.14701","title":"Scaling Laws for Autoregressive Generative Modeling","created":"2020-10-28","cited_by_count":522},{"id":"2310.16944","title":"Zephyr: Direct Distillation of LM Alignment","created":"2023-10-25","cited_by_count":506},{"id":"1802.09568","title":"Shampoo: Preconditioned Stochastic Tensor Optimization","created":"2018-02-26","cited_by_count":426},{"id":"2305.16264","title":"Scaling Data-Constrained Language Models","created":"2023-05-25","cited_by_count":403},{"id":"1711.09846","title":"Population Based Training of Neural Networks","created":"2017-11-27","cited_by_count":367},{"id":"2501.00656","title":"2 OLMo 2 Furious","created":"2024-12-31","cited_by_count":340},{"id":"2502.02737","title":"SmolLM2: When Smol Goes Big -- Data-Centric Training of a Small Language Model","created":"2025-02-04","cited_by_count":335},{"id":"2409.02060","title":"OLMoE: Open Mixture-of-Experts Language Models","created":"2024-09-03","cited_by_count":301},{"id":"1812.06162","title":"An Empirical Model of Large-Batch Training","created":"2018-12-14","cited_by_count":281},{"id":"2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","created":"2023-05-23","cited_by_count":265},{"id":"2106.04426","title":"Hash Layers For Large Sparse Models","created":"2021-06-08","cited_by_count":237},{"id":"2409.11321","title":"SOAP: Improving and Stabilizing Shampoo using Adam","created":"2024-09-17","cited_by_count":192},{"id":"2011.14522","title":"Feature Learning in Infinite-Width Neural Networks","created":"2020-11-30","cited_by_count":190},{"id":"2309.14322","title":"Small-scale proxies for large-scale Transformer training instabilities","created":"2023-09-25","cited_by_count":183},{"id":"2402.07871","title":"Scaling Laws for Fine-Grained Mixture of Experts","created":"2024-02-12","cited_by_count":173},{"id":"2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","created":"2024-05-28","cited_by_count":170},{"id":"2405.15682","title":"The Road Less Scheduled","created":"2024-05-24","cited_by_count":163},{"id":"2209.06640","title":"Revisiting Neural Scaling Laws in Language and Vision","created":"2022-09-13","cited_by_count":132},{"id":"1907.04164","title":"Which Algorithmic Choices Matter at Which Batch Sizes? Insights From a Noisy Quadratic Model","created":"2019-07-09","cited_by_count":117},{"id":"2210.14891","title":"Broken Neural Scaling Laws","created":"2022-10-26","cited_by_count":109},{"id":"2304.03208","title":"Cerebras-GPT: Open Compute-Optimal Language Models Trained on the Cerebras Wafer-Scale Cluster","created":"2023-04-06","cited_by_count":109},{"id":"2310.02244","title":"Tensor Programs VI: Feature Learning in Infinite-Depth Neural Networks","created":"2023-10-03","cited_by_count":97},{"id":"2411.04330","title":"Scaling Laws for Precision","created":"2024-11-07","cited_by_count":95},{"id":"1406.3896","title":"Freeze-Thaw Bayesian Optimization","created":"2014-06-16","cited_by_count":90},{"id":"2406.19146","title":"Resolving Discrepancies in Compute-Optimal Scaling of Language Models","created":"2024-06-27","cited_by_count":88},{"id":"2410.05192","title":"Understanding Warmup-Stable-Decay Learning Rates: A River Valley Loss Landscape Perspective","created":"2024-10-07","cited_by_count":81},{"id":"2505.01618","title":"Don't be lazy: CompleteP enables compute-efficient deep transformers","created":"2025-05-02","cited_by_count":78},{"id":"2410.21676","title":"How Does Critical Batch Size Scale in Pre-training?","created":"2024-10-29","cited_by_count":73},{"id":"2310.04415","title":"Why Do We Need Weight Decay in Modern Deep Learning?","created":"2023-10-06","cited_by_count":72},{"id":"2404.10102","title":"Chinchilla Scaling: A replication attempt","created":"2024-04-15","cited_by_count":72},{"id":"2308.16149","title":"Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models","created":"2023-08-30","cited_by_count":69},{"id":"2405.19327","title":"MAP-Neo: Highly Capable and Transparent Bilingual Large Language Model Series","created":"2024-05-29","cited_by_count":67},{"id":"2502.08606","title":"Distillation Scaling Laws","created":"2025-02-12","cited_by_count":67},{"id":"2309.16620","title":"Depthwise Hyperparameter Transfer in Residual Networks: Dynamics and Scaling Limit","created":"2023-09-28","cited_by_count":66},{"id":"2503.19206","title":"Overtrained Language Models Are Harder to Fine-Tune","created":"2025-03-24","cited_by_count":58},{"id":"2304.09871","title":"A Theory on Adam Instability in Large-Scale Machine Learning","created":"2023-04-19","cited_by_count":56},{"id":"2310.07831","title":"Optimal Linear Decay Learning Rate Schedules and Further Refinements","created":"2023-10-11","cited_by_count":43},{"id":"2405.13698","title":"How to set AdamW's weight decay as you scale model and dataset size","created":"2024-05-22","cited_by_count":43},{"id":"2505.13738","title":"Power Lines: Scaling Laws for Weight Decay and Batch Size in LLM Pre-training","created":"2025-05-19","cited_by_count":43},{"id":"2305.18411","title":"Feature-Learning Networks Are Consistent Across Widths At Realistic Scales","created":"2023-05-28","cited_by_count":41},{"id":"2409.19913","title":"Scaling Optimal LR Across Token Horizons","created":"2024-09-30","cited_by_count":40},{"id":"1807.06906","title":"Towards Automated Deep Learning: Efficient Joint Neural Architecture and Hyperparameter Search","created":"2018-07-18","cited_by_count":38},{"id":"2502.15938","title":"Straight to Zero: Why Linearly Decaying the Learning Rate to Zero Works Best for LLMs","created":"2025-02-21","cited_by_count":35},{"id":"2407.07890","title":"Training on the Test Task Confounds Evaluation and Emergence","created":"2024-07-10","cited_by_count":34},{"id":"2501.18965","title":"The Surprising Agreement Between Convex Optimization Theory and Learning-Rate Scheduling for Large Model Training","created":"2025-01-31","cited_by_count":33},{"id":"2502.05172","title":"Joint MoE Scaling Laws: Mixture of Experts Can Be Memory Efficient","created":"2025-02-07","cited_by_count":31},{"id":"2408.11029","title":"Scaling Law with Learning Rate Annealing","created":"2024-08-20","cited_by_count":29},{"id":"2408.13359","title":"Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler","created":"2024-08-23","cited_by_count":28},{"id":"2412.15285","title":"Maximize Your Data's Potential: Enhancing LLM Accuracy with Two-Phase Pretraining","created":"2024-12-18","cited_by_count":26},{"id":"2402.17457","title":"Super Consistency of Neural Network Landscapes and Learning Rate Transfer","created":"2024-02-27","cited_by_count":23},{"id":"2409.15156","title":"Rethinking Conventional Wisdom in Machine Learning: From Generalization to Scaling","created":"2024-09-23","cited_by_count":23},{"id":"2311.02076","title":"Universal Sharpness Dynamics in Neural Network Training: Fixed Point Analysis, Edge of Stability, and Route to Chaos","created":"2023-11-03","cited_by_count":22},{"id":"2410.23922","title":"Analyzing & Reducing the Need for Learning Rate Warmup in GPT Training","created":"2024-10-31","cited_by_count":22},{"id":"2503.12811","title":"A Multi-Power Law for Loss Curve Prediction Across Learning Rate Schedules","created":"2025-03-17","cited_by_count":22},{"id":"2507.02119","title":"Scaling Collapse Reveals Universal Dynamics in Compute-Optimally Trained Neural Networks","created":"2025-07-02","cited_by_count":22},{"id":"2411.15242","title":"The Zamba2 Suite: Technical Report","created":"2024-11-22","cited_by_count":21},{"id":"2309.11568","title":"BTLM-3B-8K: 7B Parameter Performance in a 3B Parameter Model","created":"2023-09-20","cited_by_count":19},{"id":"2504.15027","title":"DistilQwen2.5: Industrial Practices of Training Distilled Open Lightweight Language Models","created":"2025-04-21","cited_by_count":10},{"id":"2507.09846","title":"Through the River: Understanding the Benefit of Schedule-Free Methods for Language Model Training","created":"2025-07-14","cited_by_count":9}],"next_cursor":null}