{"total":182,"rows":[{"id":"2103.00065","title":"Gradient Descent on Neural Networks Typically Occurs at the Edge of Stability","created":"2021-02-26","cited_by_count":427,"area":null},{"id":"2306.06101","title":"Prodigy: An Expeditiously Adaptive Parameter-Free Learner","created":"2023-06-09","cited_by_count":133,"area":null},{"id":"2301.07733","title":"Learning-Rate-Free Learning by D-Adaptation","created":"2023-01-18","cited_by_count":113,"area":null},{"id":"2207.14484","title":"Adaptive Gradient Methods at the Edge of Stability","created":"2022-07-29","cited_by_count":101,"area":null},{"id":"2302.12022","title":"DoG is SGD's Best Friend: A Parameter-Free Dynamic Step Size Schedule","created":"2023-02-08","cited_by_count":85,"area":null},{"id":"2208.11195","title":"Robustness to Unbounded Smoothness of Generalized SignSGD","created":"2022-08-23","cited_by_count":81,"area":null},{"id":"2106.14343","title":"High-probability Bounds for Non-Convex Stochastic Optimization with Heavy Tails","created":"2021-06-28","cited_by_count":80,"area":null},{"id":"2411.10438","title":"MARS: Unleashing the Power of Variance Reduction for Training Large Models","created":"2024-11-15","cited_by_count":64,"area":null},{"id":"2202.05791","title":"The Power of Adaptivity in SGD: Self-Tuning Step Sizes with Unbounded Gradients and Affine Variance","created":"2022-02-11","cited_by_count":61,"area":null},{"id":"2305.18471","title":"Convergence of AdaGrad for Non-convex Objectives: Simple Proofs and Relaxed Assumptions","created":"2023-05-29","cited_by_count":57,"area":null},{"id":"2503.20762","title":"ASGO: Adaptive Structured Gradient Optimization","created":"2025-03-26","cited_by_count":57,"area":null},{"id":"2306.00204","title":"Toward Understanding Why Adam Converges Faster Than SGD for Transformers","created":"2023-05-31","cited_by_count":53,"area":null},{"id":"2112.03459","title":"A Novel Convergence Analysis for Algorithms of the Adam Family","created":"2021-12-07","cited_by_count":50,"area":null},{"id":"2302.14843","title":"High Probability Convergence of Stochastic Gradient Methods","created":"2023-02-28","cited_by_count":49,"area":null},{"id":"2310.07831","title":"Optimal Linear Decay Learning Rate Schedules and Further Refinements","created":"2023-10-11","cited_by_count":43,"area":null},{"id":"2302.06570","title":"Beyond Uniform Smoothness: A Stopped Analysis of Adaptive SGD","created":"2023-02-13","cited_by_count":41,"area":null},{"id":"2106.08208","title":"SUPER-ADAM: Faster and Universal Framework of Adaptive Gradients","created":"2021-06-15","cited_by_count":40,"area":null},{"id":"2204.02833","title":"High Probability Bounds for a Class of Nonconvex Algorithms with AdaGrad Stepsize","created":"2022-04-06","cited_by_count":40,"area":null},{"id":"2105.01650","title":"Stochastic gradient descent with noise of machine learning type. Part I: Discrete time analysis","created":"2021-05-04","cited_by_count":39,"area":null},{"id":"2202.04428","title":"Adapting to Mixing Time in Stochastic Optimization with Markovian Data","created":"2022-02-09","cited_by_count":36,"area":null},{"id":"2212.00974","title":"Faster Adaptive Federated Learning","created":"2022-12-02","cited_by_count":35,"area":null},{"id":"2305.16284","title":"DoWG Unleashed: An Efficient Universal Parameter-Free Gradient Descent Method","created":"2023-05-25","cited_by_count":35,"area":null},{"id":"2106.11396","title":"BiAdam: Fast Adaptive Bilevel Optimization Methods","created":"2021-06-21","cited_by_count":33,"area":null},{"id":"2202.07178","title":"Federated Learning with Sparsified Model Perturbation: Improving Accuracy under Client-Level Differential Privacy","created":"2022-02-15","cited_by_count":33,"area":null},{"id":"2503.10537","title":"Structured Preconditioners in Adaptive Optimization: A Unified Analysis","created":"2025-03-13","cited_by_count":33,"area":null},{"id":"2509.15816","title":"On the Convergence of Muon and Beyond","created":"2025-09-19","cited_by_count":31,"area":null},{"id":"2302.08783","title":"SGD with AdaGrad Stepsizes: Full Adaptivity with High Probability to Unknown Parameters, Unbounded Gradients and Affine Variance","created":"2023-02-17","cited_by_count":30,"area":null},{"id":"2311.03252","title":"Parameter-Agnostic Optimization under Relaxed Smoothness","created":"2023-11-06","cited_by_count":29,"area":null},{"id":"2101.11075","title":"Adaptivity without Compromise: A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic Optimization","created":"2021-01-26","cited_by_count":28,"area":null},{"id":"2402.01567","title":"Understanding Adam Optimizer via Online Learning of Updates: Adam is FTRL in Disguise","created":"2024-02-02","cited_by_count":26,"area":null},{"id":"2405.14369","title":"RoPINN: Region Optimized Physics-Informed Neural Networks","created":"2024-05-23","cited_by_count":26,"area":null},{"id":"2101.05471","title":"Towards Practical Adam: Non-Convexity, Convergence Theory, and Mini-Batch Acceleration","created":"2021-01-14","cited_by_count":25,"area":null},{"id":"2102.05320","title":"An Adaptive Stochastic Sequential Quadratic Programming with Differentiable Exact Augmented Lagrangians","created":"2021-02-10","cited_by_count":25,"area":null},{"id":"2109.05109","title":"Toward Communication Efficient Adaptive Gradient Method","created":"2021-09-10","cited_by_count":25,"area":null},{"id":"2202.01512","title":"Data Heterogeneity-Robust Federated Learning via Group Client Selection in Industrial IoT","created":"2022-02-03","cited_by_count":25,"area":null},{"id":"2206.07082","title":"Stability and Generalization of Stochastic Optimization with Nonconvex and Nonsmooth Problems","created":"2022-06-14","cited_by_count":25,"area":null},{"id":"2305.12475","title":"Two Sides of One Coin: the Limits of Untuned SGD and the Power of Adaptive Methods","created":"2023-05-21","cited_by_count":23,"area":null},{"id":"2308.06058","title":"Adaptive SGD with Polyak stepsize and Line-search: Robust Convergence and Variance Reduction","created":"2023-08-11","cited_by_count":23,"area":null},{"id":"2410.08198","title":"Adam Exploits $\\ell_\\infty$-geometry of Loss Landscape via Coordinate-wise Adaptivity","created":"2024-10-10","cited_by_count":23,"area":null},{"id":"2509.02981","title":"AdaGrad Meets Muon: Adaptive Stepsizes for Orthogonal Updates","created":"2025-09-03","cited_by_count":23,"area":null},{"id":"2406.15244","title":"AdaGrad under Anisotropic Smoothness","created":"2024-06-21","cited_by_count":22,"area":null},{"id":"2303.00565","title":"AdaSAM: Boosting Sharpness-Aware Minimization with Adaptive Learning Rate and Momentum for Training Deep Neural Networks","created":"2023-03-01","cited_by_count":21,"area":null},{"id":"2402.03982","title":"On Convergence of Adam for Stochastic Optimization under Relaxed Assumptions","created":"2024-02-06","cited_by_count":21,"area":null},{"id":"2205.13797","title":"AsyncFedED: Asynchronous Federated Learning with Euclidean Distance based Adaptive Weight Aggregation","created":"2022-05-27","cited_by_count":19,"area":null},{"id":"2301.04431","title":"Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient","created":"2023-01-11","cited_by_count":19,"area":null},{"id":"2402.07114","title":"Towards Quantifying the Preconditioning Effect of Adam","created":"2024-02-11","cited_by_count":19,"area":null},{"id":"2212.00309","title":"Differentially Private Adaptive Optimization with Delayed Preconditioners","created":"2022-12-01","cited_by_count":18,"area":null},{"id":"2411.02853","title":"ADOPT: Modified Adam Can Converge with Any $\\beta_2$ with the Optimal Rate","created":"2024-11-05","cited_by_count":18,"area":null},{"id":"2109.03194","title":"On the Convergence of Decentralized Adaptive Gradient Methods","created":"2021-09-07","cited_by_count":17,"area":null},{"id":"2109.11502","title":"Inequality Constrained Stochastic Nonlinear Optimization via Active-Set Sequential Quadratic Programming","created":"2021-09-23","cited_by_count":17,"area":null},{"id":"2206.10189","title":"A General Theory for Federated Optimization with Asynchronous and Heterogeneous Clients Updates","created":"2022-06-21","cited_by_count":17,"area":null},{"id":"2508.07473","title":"Online Convex Optimization with Heavy Tails: Old Algorithms, New Regrets, and Applications","created":"2025-08-10","cited_by_count":17,"area":null},{"id":"2108.00365","title":"A Decentralized Federated Learning Framework via Committee Mechanism with Convergence Guarantee","created":"2021-08-01","cited_by_count":16,"area":null},{"id":"2206.00743","title":"Nest Your Adaptive Algorithm for Parameter-Agnostic Nonconvex Minimax Optimization","created":"2022-06-01","cited_by_count":16,"area":null},{"id":"2406.04592","title":"Provable Complexity Improvement of AdaGrad over SGD: Upper and Lower Bounds in Stochastic Non-Convex Optimization","created":"2024-06-07","cited_by_count":16,"area":null},{"id":"2102.09700","title":"AI-SARAH: Adaptive and Implicit Stochastic Recursive Gradient Methods","created":"2021-02-19","cited_by_count":15,"area":null},{"id":"2211.01851","title":"Adaptive Stochastic Variance Reduction for Non-convex Finite-Sum Minimization","created":"2022-11-03","cited_by_count":15,"area":null},{"id":"2402.13794","title":"Revisiting Convergence of AdaGrad with Relaxed Assumptions","created":"2024-02-21","cited_by_count":15,"area":null},{"id":"2103.02062","title":"Variance Reduced Training with Stratified Sampling for Forecasting Models","created":"2021-03-02","cited_by_count":14,"area":null},{"id":"2403.15146","title":"On the Convergence of Adam under Non-uniform Smoothness: Separability from SGDM and Beyond","created":"2024-03-22","cited_by_count":14,"area":null},{"id":"2404.01436","title":"Convergence Guarantees for RMSProp and Adam in Generalized-smooth Non-convex Optimization with Affine Noise Variance","created":"2024-04-01","cited_by_count":14,"area":null},{"id":"2103.01338","title":"Acceleration via Fractal Learning Rate Schedules","created":"2021-03-01","cited_by_count":13,"area":null},{"id":"2104.14840","title":"Unified Convergence Analysis for Adaptive Optimization with Moving Average Estimator","created":"2021-04-30","cited_by_count":13,"area":null},{"id":"2106.16101","title":"AdaGDA: Faster Adaptive Gradient Descent Ascent Methods for Minimax Optimization","created":"2021-06-30","cited_by_count":13,"area":null},{"id":"2206.07553","title":"On the fast convergence of minibatch heavy ball momentum","created":"2022-06-15","cited_by_count":13,"area":null},{"id":"2209.14827","title":"On the Convergence of AdaGrad(Norm) on $\\R^{d}$: Beyond Convexity, Non-Asymptotic Rate and Acceleration","created":"2022-09-29","cited_by_count":13,"area":null},{"id":"2406.04142","title":"Stochastic Polyak Step-sizes and Momentum: Convergence Guarantees and Practical Performance","created":"2024-06-06","cited_by_count":13,"area":null},{"id":"2402.03126","title":"How Free is Parameter-Free Stochastic Optimization?","created":"2024-02-05","cited_by_count":12,"area":null},{"id":"2603.09952","title":"On the Width Scaling of Neural Optimizers Under Matrix Operator Norms I: Row/Column Normalization and Hyperparameter Transfer","created":"2026-03-10","cited_by_count":12,"area":null},{"id":"2205.14473","title":"Efficient-Adam: Communication-Efficient Distributed Adam","created":"2022-05-28","cited_by_count":11,"area":null},{"id":"2411.07724","title":"Convergence Rate Analysis of LION","created":"2024-11-12","cited_by_count":11,"area":null},{"id":"2102.07002","title":"On the Last Iterate Convergence of Momentum Methods","created":"2021-02-13","cited_by_count":10,"area":null},{"id":"2407.04358","title":"An Adaptive Stochastic Gradient Method with Non-negative Gauss-Newton Stepsizes","created":"2024-07-05","cited_by_count":10,"area":null},{"id":"2106.06075","title":"A Decentralized Adaptive Momentum Method for Solving a Class of Min-Max Optimization Problems","created":"2021-06-10","cited_by_count":9,"area":null},{"id":"2306.02527","title":"Searching for Optimal Per-Coordinate Step-sizes with Multidimensional Backtracking","created":"2023-06-05","cited_by_count":9,"area":null},{"id":"2203.01400","title":"Adaptive Gradient Methods with Local Guarantees","created":"2022-03-02","cited_by_count":8,"area":null},{"id":"2210.17478","title":"TiAda: A Time-scale Adaptive Algorithm for Nonconvex Minimax Optimization","created":"2022-10-31","cited_by_count":8,"area":null},{"id":"2211.01832","title":"Extra-Newton: A First Approach to Noise-Adaptive Accelerated Second-Order Methods","created":"2022-11-03","cited_by_count":8,"area":null},{"id":"2311.02000","title":"High Probability Convergence of Adam Under Unbounded Gradients and Affine Variance Noise","created":"2023-11-03","cited_by_count":8,"area":null},{"id":"2402.11215","title":"AdAdaGrad: Adaptive Batch Size Schemes for Adaptive Gradient Methods","created":"2024-02-17","cited_by_count":8,"area":null},{"id":"2403.02648","title":"Remove that Square Root: A New Efficient Scale-Invariant Version of AdaGrad","created":"2024-03-05","cited_by_count":8,"area":null},{"id":"2410.12455","title":"Loss Landscape Characterization of Neural Networks without Over-Parametrization","created":"2024-10-16","cited_by_count":8,"area":null},{"id":"2412.11768","title":"No More Adam: Learning Rate Scaling at Initialization is All You Need","created":"2024-12-16","cited_by_count":8,"area":null},{"id":"2505.11840","title":"On the $O(\\frac{\\sqrt{d}}{K^{1/4}})$ Convergence Rate of AdamW Measured by $\\ell_1$ Norm","created":"2025-05-17","cited_by_count":8,"area":null},{"id":"2509.26337","title":"FedMuon: Federated Learning with Bias-corrected LMO-based Optimization","created":"2025-09-30","cited_by_count":8,"area":null},{"id":"2605.06884","title":"Muon with Nesterov Momentum: Heavy-Tailed Noise and (Randomized) Inexact Polar Decomposition","created":"2026-05-07","cited_by_count":8,"area":null},{"id":"2203.15797","title":"Convergence of First-Order Methods for Constrained Nonconvex Optimization with Dependent Data","created":"2022-03-29","cited_by_count":7,"area":null},{"id":"2208.05287","title":"Adaptive Learning Rates for Faster Stochastic Gradient Methods","created":"2022-08-10","cited_by_count":7,"area":null},{"id":"2302.14796","title":"Particle-based Online Bayesian Sampling","created":"2023-02-28","cited_by_count":7,"area":null},{"id":"2306.11201","title":"Adaptive Federated Learning with Auto-Tuned Clients","created":"2023-06-19","cited_by_count":7,"area":null},{"id":"2310.02093","title":"Stochastic Gradient Descent with Preconditioned Polyak Step-size","created":"2023-10-03","cited_by_count":7,"area":null},{"id":"2403.04236","title":"Regularized DeepIV with Model Selection","created":"2024-03-07","cited_by_count":7,"area":null},{"id":"2409.04707","title":"Enhancing Deep Learning with Optimized Gradient Descent: Bridging Numerical Methods and Neural Network Training","created":"2024-09-07","cited_by_count":7,"area":null},{"id":"2109.01806","title":"On Faster Convergence of Scaled Sign Gradient Descent","created":"2021-09-04","cited_by_count":6,"area":null},{"id":"2110.04844","title":"Frequency-aware SGD for Efficient Embedding Learning with Provable Benefits","created":"2021-10-10","cited_by_count":6,"area":null},{"id":"2310.14085","title":"Adaptive, Doubly Optimal No-Regret Learning in Strongly Monotone and Exp-Concave Games with Gradient Feedback","created":"2023-10-21","cited_by_count":6,"area":null},{"id":"2110.07435","title":"Adaptive Differentially Private Empirical Risk Minimization","created":"2021-10-14","cited_by_count":5,"area":null},{"id":"2209.14853","title":"META-STORM: Generalized Fully-Adaptive Variance Reduced SGD for Unbounded Functions","created":"2022-09-29","cited_by_count":5,"area":null},{"id":"2211.07303","title":"Adaptive Federated Minimax Optimization with Lower Complexities","created":"2022-11-14","cited_by_count":5,"area":null},{"id":"2307.13680","title":"High Probability Analysis for Non-Convex Stochastic Optimization with Clipping","created":"2023-07-25","cited_by_count":5,"area":null}],"next_cursor":"100"}