{"list":"other","total":51,"rows":[{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"1706.02677","title":"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour","created":"2017-06-08","cited_by_count":1492},{"id":"1609.04836","title":"On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima","created":"2016-09-15","cited_by_count":1352},{"id":"2404.06395","title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","created":"2024-04-09","cited_by_count":723},{"id":"2401.02954","title":"DeepSeek LLM: Scaling Open-Source Language Models with Longtermism","created":"2024-01-05","cited_by_count":642},{"id":"2502.16982","title":"Muon is Scalable for LLM Training","created":"2025-02-24","cited_by_count":428},{"id":"2203.03466","title":"Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer","created":"2022-03-07","cited_by_count":288},{"id":"1812.06162","title":"An Empirical Model of Large-Batch Training","created":"2018-12-14","cited_by_count":281},{"id":"2409.20325","title":"Old Optimizer, New Norm: An Anthology","created":"2024-09-30","cited_by_count":193},{"id":"2502.07529","title":"Training Deep Learning Models with Norm-Constrained LMOs","created":"2025-02-11","cited_by_count":192},{"id":"2309.14322","title":"Small-scale proxies for large-scale Transformer training instabilities","created":"2023-09-25","cited_by_count":183},{"id":"2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","created":"2024-05-28","cited_by_count":170},{"id":"2412.02595","title":"Nemotron-CC: Transforming Common Crawl into a Refined Long-Horizon Pretraining Dataset","created":"2024-12-03","cited_by_count":160},{"id":"2506.20920","title":"FineWeb2: One Pipeline to Scale Them All -- Adapting Pre-Training Data Processing to Every Language","created":"2025-06-26","cited_by_count":143},{"id":"1710.06451","title":"A Bayesian Perspective on Generalization and Stochastic Gradient Descent","created":"2017-10-17","cited_by_count":135},{"id":"2505.16932","title":"The Polar Express: Optimal Matrix Sign Methods and Their Application to the Muon Algorithm","created":"2025-05-22","cited_by_count":110},{"id":"2310.17813","title":"A Spectral Condition for Feature Learning","created":"2023-10-26","cited_by_count":109},{"id":"2205.10287","title":"On the SDEs and Scaling Rules for Adaptive Gradient Algorithms","created":"2022-05-20","cited_by_count":104},{"id":"2509.02046","title":"Fantastic Pretraining Optimizers and Where to Find Them","created":"2025-09-02","cited_by_count":100},{"id":"2310.02244","title":"Tensor Programs VI: Feature Learning in Infinite-Depth Neural Networks","created":"2023-10-03","cited_by_count":97},{"id":"2406.19146","title":"Resolving Discrepancies in Compute-Optimal Scaling of Language Models","created":"2024-06-27","cited_by_count":88},{"id":"2410.06511","title":"TorchTitan: One-stop PyTorch native solution for production ready LLM pre-training","created":"2024-10-09","cited_by_count":85},{"id":"2505.02222","title":"Practical Efficiency of Muon for Pretraining","created":"2025-05-04","cited_by_count":80},{"id":"2505.01618","title":"Don't be lazy: CompleteP enables compute-efficient deep transformers","created":"2025-05-02","cited_by_count":78},{"id":"2410.21265","title":"Modular Duality in Deep Learning","created":"2024-10-28","cited_by_count":77},{"id":"2407.05872","title":"Scaling Exponents Across Parameterizations and Optimizers","created":"2024-07-08","cited_by_count":76},{"id":"2410.21676","title":"How Does Critical Batch Size Scale in Pre-training?","created":"2024-10-29","cited_by_count":73},{"id":"2505.13416","title":"Gluon: Making Muon & Scion Great Again! (Bridging Theory and Practice of LMO-based Optimizers for LLMs)","created":"2025-05-19","cited_by_count":72},{"id":"2410.01131","title":"nGPT: Normalized Transformer with Representation Learning on the Hypersphere","created":"2024-10-01","cited_by_count":70},{"id":"2405.14813","title":"Scalable Optimization in the Modular Norm","created":"2024-05-23","cited_by_count":68},{"id":"2504.05295","title":"Dion: Distributed Orthonormalized Updates","created":"2025-04-07","cited_by_count":68},{"id":"2509.01440","title":"Benchmarking Optimizers for Large Language Model Pretraining","created":"2025-09-01","cited_by_count":68},{"id":"2309.16620","title":"Depthwise Hyperparameter Transfer in Residual Networks: Dynamics and Scaling Limit","created":"2023-09-28","cited_by_count":66},{"id":"2507.22448","title":"Falcon-H1: A Family of Hybrid-Head Language Models Redefining Efficiency and Performance","created":"2025-07-30","cited_by_count":57},{"id":"2006.09092","title":"Learning Rates as a Function of Batch Size: A Random Matrix Theory Approach to Neural Network Training","created":"2020-06-16","cited_by_count":50},{"id":"2509.26030","title":"Muon Outperforms Adam in Tail-End Associative Memory Learning","created":"2025-09-30","cited_by_count":46},{"id":"2505.13738","title":"Power Lines: Scaling Laws for Weight Decay and Batch Size in LLM Pre-training","created":"2025-05-19","cited_by_count":43},{"id":"2409.19913","title":"Scaling Optimal LR Across Token Horizons","created":"2024-09-30","cited_by_count":40},{"id":"2110.00641","title":"Batch size-invariance for policy optimization","created":"2021-10-01","cited_by_count":35},{"id":"2502.02732","title":"Peri-LN: Revisiting Normalization Layer in the Transformer Architecture","created":"2025-02-04","cited_by_count":34},{"id":"2408.13359","title":"Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler","created":"2024-08-23","cited_by_count":28},{"id":"2507.01598","title":"Convergence Bound and Critical Batch Size of Muon Optimizer","created":"2025-07-02","cited_by_count":28},{"id":"2411.15958","title":"Adaptive Methods through the Lens of SDEs: Theoretical Insights on the Role of Noise","created":"2024-11-24","cited_by_count":23},{"id":"2405.14578","title":"Surge Phenomenon in Optimal Learning Rate and Batch Size Scaling","created":"2024-05-23","cited_by_count":22},{"id":"2507.13338","title":"Training Transformers with Enforced Lipschitz Constants","created":"2025-07-17","cited_by_count":20},{"id":"2506.01913","title":"Generalized Gradient Norm Clipping & Non-Euclidean $(L_0,L_1)$-Smoothness","created":"2025-06-02","cited_by_count":17},{"id":"2410.05838","title":"Time Transfer: On Optimal Learning Rate and Batch Size In The Infinite Data Limit","created":"2024-10-08","cited_by_count":9},{"id":"2506.05447","title":"Training Dynamics Underlying Language Model Scaling Laws: Loss Deceleration and Zero-Sum Learning","created":"2025-06-05","cited_by_count":5}],"next_cursor":null}