{"list":"other","total":54,"rows":[{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2203.02155","title":"Training language models to follow instructions with human feedback","created":"2022-03-04","cited_by_count":15971},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1804.04235","title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost","created":"2018-04-11","cited_by_count":905},{"id":"1503.05671","title":"Optimizing Neural Networks with Kronecker-factored Approximate Curvature","created":"2015-03-19","cited_by_count":796},{"id":"1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","created":"2019-04-01","cited_by_count":674},{"id":"1802.04434","title":"signSGD: Compressed Optimisation for Non-Convex Problems","created":"2018-02-13","cited_by_count":571},{"id":"2403.03507","title":"GaLore: Memory-Efficient LLM Training by Gradient Low-Rank Projection","created":"2024-03-06","cited_by_count":463},{"id":"2502.16982","title":"Muon is Scalable for LLM Training","created":"2025-02-24","cited_by_count":428},{"id":"1802.09568","title":"Shampoo: Preconditioned Stochastic Tensor Optimization","created":"2018-02-26","cited_by_count":426},{"id":"1901.09847","title":"Error Feedback Fixes SignSGD and other Gradient Compression Schemes","created":"2019-01-28","cited_by_count":310},{"id":"1706.04454","title":"Empirical Analysis of the Hessian of Over-Parametrized Neural Networks","created":"2017-06-14","cited_by_count":290},{"id":"2207.14255","title":"Efficient Training of Language Models to Fill in the Middle","created":"2022-07-28","cited_by_count":255},{"id":"1812.04754","title":"Gradient Descent Happens in a Tiny Subspace","created":"2018-12-12","cited_by_count":217},{"id":"2307.05695","title":"ReLoRA: High-Rank Training Through Low-Rank Updates","created":"2023-07-11","cited_by_count":204},{"id":"2409.20325","title":"Old Optimizer, New Norm: An Anthology","created":"2024-09-30","cited_by_count":193},{"id":"2409.11321","title":"SOAP: Improving and Stabilizing Shampoo using Adam","created":"2024-09-17","cited_by_count":192},{"id":"2502.07529","title":"Training Deep Learning Models with Norm-Constrained LMOs","created":"2025-02-11","cited_by_count":192},{"id":"1806.01811","title":"AdaGrad stepsizes: Sharp convergence over nonconvex landscapes","created":"2018-06-05","cited_by_count":182},{"id":"2301.11235","title":"Handbook of Convergence Theorems for (Stochastic) Gradient Methods","created":"2023-01-26","cited_by_count":163},{"id":"1611.07476","title":"Eigenvalues of the Hessian in Deep Learning: Singularity and Beyond","created":"2016-11-22","cited_by_count":159},{"id":"2002.03305","title":"Momentum Improves Normalized SGD","created":"2020-02-09","cited_by_count":139},{"id":"2406.16793","title":"Adam-mini: Use Fewer Learning Rates To Gain More","created":"2024-06-24","cited_by_count":137},{"id":"1806.04090","title":"ATOMO: Communication-efficient Learning via Atomic Sparsification","created":"2018-06-11","cited_by_count":136},{"id":"2402.16788","title":"Why Transformers Need Adam: A Hessian Perspective","created":"2024-02-26","cited_by_count":124},{"id":"2304.13960","title":"Noise Is Not the Main Factor Behind the Gap Between SGD and Adam on Transformers, but Sign Descent Might Be","created":"2023-04-27","cited_by_count":114},{"id":"2310.17813","title":"A Spectral Condition for Feature Learning","created":"2023-10-26","cited_by_count":109},{"id":"1910.05446","title":"On Empirical Comparisons of Optimizers for Deep Learning","created":"2019-10-11","cited_by_count":103},{"id":"2502.02900","title":"A Note on the Convergence of Muon","created":"2025-02-05","cited_by_count":83},{"id":"2208.11195","title":"Robustness to Unbounded Smoothness of Generalized SignSGD","created":"2022-08-23","cited_by_count":81},{"id":"2309.06497","title":"A Distributed Data-Parallel PyTorch Implementation of the Distributed Shampoo Optimizer for Training Neural Networks At-Scale","created":"2023-09-12","cited_by_count":77},{"id":"2410.21265","title":"Modular Duality in Deep Learning","created":"2024-10-28","cited_by_count":77},{"id":"2405.14813","title":"Scalable Optimization in the Modular Norm","created":"2024-05-23","cited_by_count":68},{"id":"2411.10438","title":"MARS: Unleashing the Power of Variance Reduction for Training Large Models","created":"2024-11-15","cited_by_count":64},{"id":"2406.17748","title":"A New Perspective on Shampoo's Preconditioner","created":"2024-06-25","cited_by_count":59},{"id":"2412.05270","title":"APOLLO: SGD-like Memory, AdamW-level Performance","created":"2024-12-06","cited_by_count":54},{"id":"1809.02864","title":"Online Adaptive Methods, Universality and Acceleration","created":"2018-09-08","cited_by_count":51},{"id":"2010.04261","title":"Dissecting Hessian: Understanding Common Structure of Hessian in Neural Networks","created":"2020-10-08","cited_by_count":45},{"id":"2503.10537","title":"Structured Preconditioners in Adaptive Optimization: A Unified Analysis","created":"2025-03-13","cited_by_count":33},{"id":"2502.17410","title":"COSMOS: A Hybrid Adaptive Optimizer for Memory-Efficient Training of LLMs","created":"2025-02-24","cited_by_count":30},{"id":"1908.10525","title":"Linear Convergence of Adaptive Stochastic Gradient Descent","created":"2019-08-28","cited_by_count":29},{"id":"2110.12661","title":"ZerO Initialization: Initializing Neural Networks with only Zeros and Ones","created":"2021-10-25","cited_by_count":28},{"id":"2402.03496","title":"Can We Remove the Square-Root in Adaptive Gradient Methods? A Second-Order Perspective","created":"2024-02-05","cited_by_count":25},{"id":"2106.02925","title":"Tensor Normal Training for Deep Learning Models","created":"2021-06-05","cited_by_count":24},{"id":"2410.08198","title":"Adam Exploits $\\ell_\\infty$-geometry of Loss Landscape via Coordinate-wise Adaptivity","created":"2024-10-10","cited_by_count":23},{"id":"2406.15244","title":"AdaGrad under Anisotropic Smoothness","created":"2024-06-21","cited_by_count":22},{"id":"1706.06569","title":"A Unified Approach to Adaptive Regularization in Online and Stochastic Optimization","created":"2017-06-20","cited_by_count":19},{"id":"2302.03764","title":"Sketchy: Memory-efficient Adaptive Regularization with Frequent Directions","created":"2023-02-07","cited_by_count":17},{"id":"2202.04124","title":"A Mini-Block Fisher Method for Deep Neural Networks","created":"2022-02-08","cited_by_count":8},{"id":"2410.09568","title":"Second-Order Min-Max Optimization with Lazy Hessians","created":"2024-10-12","cited_by_count":5},{"id":"2502.07488","title":"Improving Adaptive Moment Optimization via Preconditioner Diagonalization","created":"2025-02-11","cited_by_count":5}],"next_cursor":null}