{"total":81,"rows":[{"id":"2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","created":"2023-05-23","cited_by_count":265,"area":null},{"id":"2402.16788","title":"Why Transformers Need Adam: A Hessian Perspective","created":"2024-02-26","cited_by_count":124,"area":null},{"id":"2304.13960","title":"Noise Is Not the Main Factor Behind the Gap Between SGD and Adam on Transformers, but Sign Descent Might Be","created":"2023-04-27","cited_by_count":114,"area":null},{"id":"2304.13972","title":"Convergence of Adam Under Relaxed Assumptions","created":"2023-04-27","cited_by_count":99,"area":null},{"id":"2310.01082","title":"Linear attention is (maybe) all you need (to understand transformer optimization)","created":"2023-10-02","cited_by_count":98,"area":null},{"id":"2402.19449","title":"Heavy-Tailed Class Imbalance and Why Adam Outperforms Gradient Descent on Language Models","created":"2024-02-29","cited_by_count":80,"area":null},{"id":"2505.13416","title":"Gluon: Making Muon & Scion Great Again! (Bridging Theory and Practice of LMO-based Optimizers for LLMs)","created":"2025-05-19","cited_by_count":72,"area":null},{"id":"2404.04454","title":"Implicit Bias of AdamW: $\\ell_\\infty$ Norm Constrained Optimization","created":"2024-04-05","cited_by_count":58,"area":null},{"id":"2305.18471","title":"Convergence of AdaGrad for Non-convex Objectives: Simple Proofs and Relaxed Assumptions","created":"2023-05-29","cited_by_count":57,"area":null},{"id":"2503.20762","title":"ASGO: Adaptive Structured Gradient Optimization","created":"2025-03-26","cited_by_count":57,"area":null},{"id":"2310.05898","title":"Lion Secretly Solves Constrained Optimization: As Lyapunov Predicts","created":"2023-10-09","cited_by_count":51,"area":null},{"id":"2410.13849","title":"From Gradient Clipping to Normalization for Heavy Tailed SGD","created":"2024-10-17","cited_by_count":49,"area":null},{"id":"2306.01264","title":"Convex and Non-convex Optimization Under Generalized Smoothness","created":"2023-06-02","cited_by_count":44,"area":null},{"id":"2412.19529","title":"Nonconvex Stochastic Optimization under Heavy-Tailed Noises: Optimal Convergence without Gradient Clipping","created":"2024-12-27","cited_by_count":42,"area":null},{"id":"2302.06570","title":"Beyond Uniform Smoothness: A Stopped Analysis of Adaptive SGD","created":"2023-02-13","cited_by_count":41,"area":null},{"id":"2506.04192","title":"Lions and Muons: Optimization via Stochastic Frank-Wolfe under Heavy-Tailed Noise","created":"2025-06-04","cited_by_count":41,"area":null},{"id":"2310.17998","title":"Closing the Gap Between the Upper Bound and the Lower Bound of Adam's Iteration Complexity","created":"2023-10-27","cited_by_count":38,"area":null},{"id":"2405.16002","title":"Does SGD really happen in tiny subspaces?","created":"2024-05-25","cited_by_count":36,"area":null},{"id":"2311.03252","title":"Parameter-Agnostic Optimization under Relaxed Smoothness","created":"2023-11-06","cited_by_count":29,"area":null},{"id":"2402.01567","title":"Understanding Adam Optimizer via Online Learning of Updates: Adam is FTRL in Disguise","created":"2024-02-02","cited_by_count":26,"area":null},{"id":"2405.18199","title":"Adam with model exponential moving average is effective for nonconvex optimization","created":"2024-05-28","cited_by_count":23,"area":null},{"id":"2303.00883","title":"Variance-reduced Clipping for Non-convex Optimization","created":"2023-03-02","cited_by_count":22,"area":null},{"id":"2406.15244","title":"AdaGrad under Anisotropic Smoothness","created":"2024-06-21","cited_by_count":22,"area":null},{"id":"2402.03982","title":"On Convergence of Adam for Stochastic Optimization under Relaxed Assumptions","created":"2024-02-06","cited_by_count":21,"area":null},{"id":"2412.13148","title":"SWAN: SGD with Normalization and Whitening Enables Stateless LLM Training","created":"2024-12-17","cited_by_count":20,"area":null},{"id":"2402.07114","title":"Towards Quantifying the Preconditioning Effect of Adam","created":"2024-02-11","cited_by_count":19,"area":null},{"id":"2505.24749","title":"SUMO: Subspace-Aware Moment-Orthogonalization for Accelerating Memory-Efficient LLM Training","created":"2025-05-30","cited_by_count":19,"area":null},{"id":"2404.00438","title":"Communication Efficient Distributed Training with Distributed Lion","created":"2024-03-30","cited_by_count":18,"area":null},{"id":"2506.01913","title":"Generalized Gradient Norm Clipping & Non-Euclidean $(L_0,L_1)$-Smoothness","created":"2025-06-02","cited_by_count":17,"area":null},{"id":"2406.04592","title":"Provable Complexity Improvement of AdaGrad over SGD: Upper and Lower Bounds in Stochastic Non-Convex Optimization","created":"2024-06-07","cited_by_count":16,"area":null},{"id":"2401.09587","title":"Bilevel Optimization under Unbounded Smoothness: A New Algorithm and Convergence Analysis","created":"2024-01-17","cited_by_count":15,"area":null},{"id":"2402.13794","title":"Revisiting Convergence of AdaGrad with Relaxed Assumptions","created":"2024-02-21","cited_by_count":15,"area":null},{"id":"2403.15146","title":"On the Convergence of Adam under Non-uniform Smoothness: Separability from SGDM and Beyond","created":"2024-03-22","cited_by_count":14,"area":null},{"id":"2404.01436","title":"Convergence Guarantees for RMSProp and Adam in Generalized-smooth Non-convex Optimization with Affine Noise Variance","created":"2024-04-01","cited_by_count":14,"area":null},{"id":"2502.07923","title":"Sign Operator for Coping with Heavy-Tailed Noise in Non-Convex Optimization: High Probability Bounds Under $(L_0, L_1)$-Smoothness","created":"2025-02-11","cited_by_count":13,"area":null},{"id":"2408.09074","title":"Gradient-Variation Online Learning under Generalized Smoothness","created":"2024-08-17","cited_by_count":11,"area":null},{"id":"2410.19964","title":"Understanding Adam Requires Better Rotation Dependent Assumptions","created":"2024-10-25","cited_by_count":11,"area":null},{"id":"2409.19212","title":"An Accelerated Algorithm for Stochastic Bilevel Optimization under Unbounded Smoothness","created":"2024-09-28","cited_by_count":9,"area":null},{"id":"2505.11840","title":"On the $O(\\frac{\\sqrt{d}}{K^{1/4}})$ Convergence Rate of AdamW Measured by $\\ell_1$ Norm","created":"2025-05-17","cited_by_count":8,"area":null},{"id":"2310.18784","title":"High-probability Convergence Bounds for Nonlinear Stochastic Gradient Descent Under Heavy-tailed Noise","created":"2023-10-28","cited_by_count":7,"area":null},{"id":"2405.19440","title":"MGDA Converges under Generalized Smoothness, Provably","created":"2024-05-29","cited_by_count":7,"area":null},{"id":"2410.15637","title":"Large Deviation Upper Bounds and Improved MSE Rates of Nonlinear SGD: Heavy-tailed Noise and Power of Symmetry","created":"2024-10-21","cited_by_count":7,"area":null},{"id":"2412.20017","title":"A Nearly Optimal Single Loop Algorithm for Stochastic Bilevel Optimization under Unbounded Smoothness","created":"2024-12-28","cited_by_count":7,"area":null},{"id":"2510.17802","title":"Unbiased Gradient Low-Rank Projection","created":"2025-10-20","cited_by_count":7,"area":null},{"id":"2602.15322","title":"On Surprising Effectiveness of Masking Updates in Adaptive Optimizers","created":"2026-02-17","cited_by_count":7,"area":null},{"id":"2602.21545","title":"MUON+: Towards More Effective Muon via One Additional Normalization Step for LLM Pre-training","created":"2026-02-25","cited_by_count":6,"area":null},{"id":"2407.11654","title":"R-SFLLM: Jamming Resilient Framework for Split Federated Learning with Large Language Models","created":"2024-07-16","cited_by_count":5,"area":null},{"id":"2604.15416","title":"StoSignSGD: Unbiased Structural Stochasticity Fixes SignSGD for Training Large Language Models","created":"2026-04-16","cited_by_count":5,"area":null},{"id":"2605.06654","title":"Optimizer-Model Consistency: Full Finetuning with the Same Optimizer as Pretraining Forgets Less","created":"2026-05-07","cited_by_count":5,"area":null},{"id":"2310.14616","title":"Rethinking SIGN Training: Provable Nonconvex Acceleration without First- and Second-Order Gradient Lipschitz","created":"2023-10-23","cited_by_count":4,"area":null},{"id":"2410.14054","title":"Adaptive Gradient Normalization and Independent Sampling for (Stochastic) Generalized-Smooth Optimization","created":"2024-10-17","cited_by_count":4,"area":null},{"id":"2410.16871","title":"Error Feedback under $(L_0,L_1)$-Smoothness: Normalization and Momentum","created":"2024-10-22","cited_by_count":4,"area":null},{"id":"2506.00181","title":"On the Interaction of Batch Noise, Adaptivity, and Compression, under $(L_0,L_1)$-Smoothness: An SDE Approach","created":"2025-05-30","cited_by_count":4,"area":null},{"id":"2507.05966","title":"Simple Convergence Proof of Adam From a Sign-like Descent Perspective","created":"2025-07-08","cited_by_count":4,"area":null},{"id":"2509.07972","title":"Theoretical Analysis on how Learning Rate Warmup Accelerates Convergence","created":"2025-09-09","cited_by_count":4,"area":null},{"id":"2603.02064","title":"Never Saddle for Reparameterized Steepest Descent as Mirror Flow","created":"2026-03-02","cited_by_count":4,"area":null},{"id":"2405.20127","title":"SPAM: Stochastic Proximal Point Method with Momentum Variance Reduction for Non-convex Cross-Device Federated Learning","created":"2024-05-30","cited_by_count":3,"area":null},{"id":"2505.24399","title":"LightSAM: Parameter-Agnostic Sharpness-Aware Minimization","created":"2025-05-30","cited_by_count":3,"area":null},{"id":"2310.15976","title":"Convergence of Sign-based Random Reshuffling Algorithms for Nonconvex Optimization","created":"2023-10-24","cited_by_count":2,"area":null},{"id":"2605.06615","title":"When and Why SignSGD Outperforms SGD: A Theoretical Study Based on $\\ell_1$-norm Lower Bounds","created":"2026-05-07","cited_by_count":2,"area":null},{"id":"2605.18174","title":"Ringmaster LMO: Asynchronous Linear Minimization Oracle Momentum Method","created":"2026-05-18","cited_by_count":2,"area":null},{"id":"2605.19811","title":"LionMuon: Alternating Spectral and Sign Descent for Efficient Training","created":"2026-05-19","cited_by_count":2,"area":null},{"id":"2606.14560","title":"Free Heavy-Tailed Lunch for Muon: A Theoretical Justification of Empirical Success","created":"2026-06-12","cited_by_count":2,"area":null},{"id":"2410.13954","title":"Nonlinear Stochastic Gradient Descent and Heavy-tailed Noise: A Unified Framework and High-probability Guarantees","created":"2024-10-17","cited_by_count":1,"area":null},{"id":"2503.03908","title":"On the Convergence of Adam-Type Algorithm for Bilevel Optimization under Unbounded Smoothness","created":"2025-03-05","cited_by_count":1,"area":null},{"id":"2508.06884","title":"Near-Optimal Convergence of Accelerated Gradient Methods under Generalized and $(L_0, L_1)$-Smoothness","created":"2025-08-09","cited_by_count":1,"area":null},{"id":"2607.04233","title":"Unified convergence analysis for gradient descent optimization methods in the training of deep neural networks","created":"2026-07-05","cited_by_count":1,"area":null},{"id":"2209.05372","title":"Convergence of Batch Updating Methods with Approximate Gradients and/or Noisy Measurements: Theory and Computational Results","created":"2022-09-12","cited_by_count":0,"area":null},{"id":"2306.10278","title":"Adaptive Strategies in Non-convex Optimization","created":"2023-06-17","cited_by_count":0,"area":null},{"id":"2402.00389","title":"On the $O(\\frac{\\sqrt{d}}{T^{1/4}})$ Convergence Rate of RMSProp and Its Momentum Extension Measured by $\\ell_1$ Norm","created":"2024-02-01","cited_by_count":0,"area":null},{"id":"2410.05807","title":"Extended convexity and smoothness and their applications in deep learning","created":"2024-10-08","cited_by_count":0,"area":null},{"id":"2507.08913","title":"Revisiting Convergence: Shuffling Complexity Beyond Lipschitz Smoothness","created":"2025-07-11","cited_by_count":0,"area":null},{"id":"2509.08726","title":"Decentralized Stochastic Nonconvex Optimization under the $(L_0,L_1)$-Smoothness","created":"2025-09-10","cited_by_count":0,"area":null},{"id":"2512.13034","title":"Alada: Alternating Adaptation of Momentum Method for Memory-Efficient Matrix Optimization","created":"2025-12-15","cited_by_count":0,"area":null},{"id":"2601.03566","title":"Provably Convergent Decentralized Optimization over Directed Graphs under Generalized Smoothness","created":"2026-01-07","cited_by_count":0,"area":null},{"id":"2603.12512","title":"Byzantine-Robust Optimization under $(L_0, L_1)$-Smoothness","created":"2026-03-12","cited_by_count":0,"area":null},{"id":"2605.15522","title":"Stochastic Non-Smooth Convex Optimization with Unbounded Gradients","created":"2026-05-15","cited_by_count":0,"area":null},{"id":"2609.23055","title":"Optimizers for Diffusion Models: A Controlled Benchmark","created":"2026-09-19","cited_by_count":0,"area":"diffusion"},{"id":"2609.30276","title":"Why Clipping Matters in AdaGrad? Toward a High-Probability Theory under Generalized Smoothness","created":"2026-08-12","cited_by_count":0,"area":null},{"id":"2609.35297","title":"LionMuon: Alternating Spectral and Sign Descent for Efficient Training","created":"2026-09-28","cited_by_count":0,"area":null},{"id":"2609.37787","title":"Adam under Generalized Smoothness with Second-Moment-Type Stochastic Gradients","created":"2026-09-29","cited_by_count":0,"area":null}],"next_cursor":null}