{"total":341,"rows":[{"id":"2103.00065","title":"Gradient Descent on Neural Networks Typically Occurs at the Edge of Stability","created":"2021-02-26","cited_by_count":427,"area":null},{"id":"2102.06171","title":"High-Performance Large-Scale Image Recognition Without Normalization","created":"2021-02-11","cited_by_count":360,"area":null},{"id":"2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","created":"2023-05-23","cited_by_count":265,"area":null},{"id":"2406.16793","title":"Adam-mini: Use Fewer Learning Rates To Gain More","created":"2024-06-24","cited_by_count":137,"area":null},{"id":"2402.16788","title":"Why Transformers Need Adam: A Hessian Perspective","created":"2024-02-26","cited_by_count":124,"area":null},{"id":"2304.13960","title":"Noise Is Not the Main Factor Behind the Gap Between SGD and Adam on Transformers, but Sign Descent Might Be","created":"2023-04-27","cited_by_count":114,"area":null},{"id":"2304.13972","title":"Convergence of Adam Under Relaxed Assumptions","created":"2023-04-27","cited_by_count":99,"area":null},{"id":"2310.01082","title":"Linear attention is (maybe) all you need (to understand transformer optimization)","created":"2023-10-02","cited_by_count":98,"area":null},{"id":"2206.04817","title":"The Slingshot Mechanism: An Empirical Study of Adaptive Optimizers and the Grokking Phenomenon","created":"2022-06-10","cited_by_count":94,"area":null},{"id":"2503.12645","title":"Understanding Gradient Orthogonalization for Deep Learning via Non-Euclidean Trust-Region Optimization","created":"2025-03-16","cited_by_count":92,"area":null},{"id":"2305.01588","title":"Revisiting Gradient Clipping: Stochastic bias and tight convergence guarantees","created":"2023-05-02","cited_by_count":83,"area":null},{"id":"2208.11195","title":"Robustness to Unbounded Smoothness of Generalized SignSGD","created":"2022-08-23","cited_by_count":81,"area":null},{"id":"2106.14343","title":"High-probability Bounds for Non-Convex Stochastic Optimization with Heavy Tails","created":"2021-06-28","cited_by_count":80,"area":null},{"id":"2402.19449","title":"Heavy-Tailed Class Imbalance and Why Adam Outperforms Gradient Descent on Language Models","created":"2024-02-29","cited_by_count":80,"area":null},{"id":"2505.13416","title":"Gluon: Making Muon & Scion Great Again! (Bridging Theory and Practice of LMO-based Optimizers for LLMs)","created":"2025-05-19","cited_by_count":72,"area":null},{"id":"2405.14813","title":"Scalable Optimization in the Modular Norm","created":"2024-05-23","cited_by_count":68,"area":null},{"id":"2109.14119","title":"Stochastic Training is Not Necessary for Generalization","created":"2021-09-29","cited_by_count":63,"area":null},{"id":"2205.07536","title":"Reachability Constrained Reinforcement Learning","created":"2022-05-16","cited_by_count":59,"area":null},{"id":"2305.18471","title":"Convergence of AdaGrad for Non-convex Objectives: Simple Proofs and Relaxed Assumptions","created":"2023-05-29","cited_by_count":57,"area":null},{"id":"2306.00204","title":"Toward Understanding Why Adam Converges Faster Than SGD for Transformers","created":"2023-05-31","cited_by_count":53,"area":null},{"id":"2105.11982","title":"Quantifying Uncertainty in Deep Spatiotemporal Forecasting","created":"2021-05-25","cited_by_count":48,"area":null},{"id":"2203.05363","title":"Differentially Private Learning Needs Hidden State (Or Much Faster Convergence)","created":"2022-03-10","cited_by_count":48,"area":null},{"id":"2403.09508","title":"SkateFormer: Skeletal-Temporal Transformer for Human Action Recognition","created":"2024-03-14","cited_by_count":47,"area":null},{"id":"2105.06072","title":"Leveraging Non-uniformity in First-order Non-convex Optimization","created":"2021-05-13","cited_by_count":44,"area":null},{"id":"2306.01264","title":"Convex and Non-convex Optimization Under Generalized Smoothness","created":"2023-06-02","cited_by_count":44,"area":null},{"id":"2410.01623","title":"Fira: Can We Achieve Full-rank Training of LLMs Under Low-rank Constraint?","created":"2024-10-02","cited_by_count":43,"area":null},{"id":"2412.19529","title":"Nonconvex Stochastic Optimization under Heavy-Tailed Noises: Optimal Convergence without Gradient Clipping","created":"2024-12-27","cited_by_count":42,"area":null},{"id":"2302.06570","title":"Beyond Uniform Smoothness: A Stopped Analysis of Adaptive SGD","created":"2023-02-13","cited_by_count":41,"area":null},{"id":"2212.04055","title":"Mitigating Memorization of Noisy Labels by Clipping the Model Prediction","created":"2022-12-08","cited_by_count":40,"area":null},{"id":"2505.21829","title":"In Search of Adam's Secret Sauce","created":"2025-05-27","cited_by_count":40,"area":null},{"id":"2110.12459","title":"Non-convex Distributionally Robust Optimization: Non-asymptotic Analysis","created":"2021-10-24","cited_by_count":39,"area":null},{"id":"2102.06489","title":"Stability and Convergence of Stochastic Gradient Clipping: Beyond Lipschitz Continuity and Smoothness","created":"2021-02-12","cited_by_count":37,"area":null},{"id":"2206.13033","title":"Normalized/Clipped SGD with Perturbation for Differentially Private Non-Convex Optimization","created":"2022-06-27","cited_by_count":36,"area":null},{"id":"2305.16284","title":"DoWG Unleashed: An Efficient Universal Parameter-Free Gradient Descent Method","created":"2023-05-25","cited_by_count":35,"area":null},{"id":"2102.10264","title":"On Proximal Policy Optimization's Heavy-tailed Gradients","created":"2021-02-20","cited_by_count":31,"area":null},{"id":"2110.03128","title":"On the Generalization of Models Trained with SGD: Information-Theoretic Bounds and Implications","created":"2021-10-07","cited_by_count":31,"area":null},{"id":"2208.09900","title":"Provable Adaptivity of Adam under Non-uniform Smoothness","created":"2022-08-21","cited_by_count":30,"area":null},{"id":"2202.00980","title":"Robust Training of Neural Networks Using Scale Invariant Architectures","created":"2022-02-02","cited_by_count":29,"area":null},{"id":"2311.03252","title":"Parameter-Agnostic Optimization under Relaxed Smoothness","created":"2023-11-06","cited_by_count":29,"area":null},{"id":"2110.11222","title":"Is High Variance Unavoidable in RL? A Case Study in Continuous Control","created":"2021-10-21","cited_by_count":28,"area":null},{"id":"2207.06983","title":"Multitrack Music Transformer","created":"2022-07-14","cited_by_count":28,"area":null},{"id":"2111.07695","title":"Joint Synthesis of Safety Certificate and Safe Control Policy using Constrained Reinforcement Learning","created":"2021-11-15","cited_by_count":27,"area":null},{"id":"2206.13951","title":"Robustifying Vision Transformer without Retraining from Scratch by Test-Time Class-Conditional Feature Alignment","created":"2022-06-28","cited_by_count":27,"area":null},{"id":"2405.04669","title":"Towards a Theoretical Understanding of the 'Reversal Curse' via Training Dynamics","created":"2024-05-07","cited_by_count":27,"area":"lr"},{"id":"2510.19933","title":"Beyond the Ideal: Analyzing the Inexact Muon Update","created":"2025-10-22","cited_by_count":27,"area":null},{"id":"2402.01567","title":"Understanding Adam Optimizer via Online Learning of Updates: Adam is FTRL in Disguise","created":"2024-02-02","cited_by_count":26,"area":null},{"id":"2106.05956","title":"Beyond BatchNorm: Towards a Unified Understanding of Normalization in Deep Learning","created":"2021-06-10","cited_by_count":25,"area":null},{"id":"2212.07065","title":"CLIPSep: Learning Text-queried Sound Separation with Noisy Unlabeled Videos","created":"2022-12-14","cited_by_count":23,"area":null},{"id":"2305.12475","title":"Two Sides of One Coin: the Limits of Untuned SGD and the Power of Adaptive Methods","created":"2023-05-21","cited_by_count":23,"area":null},{"id":"2405.18199","title":"Adam with model exponential moving average is effective for nonconvex optimization","created":"2024-05-28","cited_by_count":23,"area":null},{"id":"2410.08198","title":"Adam Exploits $\\ell_\\infty$-geometry of Loss Landscape via Coordinate-wise Adaptivity","created":"2024-10-10","cited_by_count":23,"area":null},{"id":"2507.17634","title":"WSM: Decay-Free Learning Rate Schedule via Checkpoint Merging for LLM Pre-training","created":"2025-07-23","cited_by_count":23,"area":null},{"id":"2106.13731","title":"Ranger21: a synergistic deep learning optimizer","created":"2021-06-25","cited_by_count":22,"area":null},{"id":"2303.00883","title":"Variance-reduced Clipping for Non-convex Optimization","created":"2023-03-02","cited_by_count":22,"area":null},{"id":"2305.03938","title":"Adam-family Methods for Nonsmooth Optimization with Convergence Guarantees","created":"2023-05-06","cited_by_count":22,"area":null},{"id":"2406.15244","title":"AdaGrad under Anisotropic Smoothness","created":"2024-06-21","cited_by_count":22,"area":null},{"id":"2509.14562","title":"LiMuon: Light and Fast Muon Optimizer for Large Models","created":"2025-09-18","cited_by_count":22,"area":null},{"id":"2212.00328","title":"Differentially Private Learning with Per-Sample Adaptive Clipping","created":"2022-12-01","cited_by_count":21,"area":null},{"id":"2310.18605","title":"TorchDEQ: A Library for Deep Equilibrium Models","created":"2023-10-28","cited_by_count":21,"area":null},{"id":"2311.04161","title":"Breaking the Heavy-Tailed Noise Barrier in Stochastic Optimization Problems","created":"2023-11-07","cited_by_count":21,"area":null},{"id":"2402.03982","title":"On Convergence of Adam for Stochastic Optimization under Relaxed Assumptions","created":"2024-02-06","cited_by_count":21,"area":null},{"id":"2205.05040","title":"A Communication-Efficient Distributed Gradient Clipping Algorithm for Training Deep Neural Networks","created":"2022-05-10","cited_by_count":20,"area":null},{"id":"2205.14375","title":"WaveMix: A Resource-efficient Neural Network for Image Analysis","created":"2022-05-28","cited_by_count":19,"area":null},{"id":"2101.09379","title":"SGD-Net: Efficient Model-Based Deep Learning with Theoretical Guarantees","created":"2021-01-22","cited_by_count":18,"area":null},{"id":"2102.08166","title":"Differential Privacy and Byzantine Resilience in SGD: Do They Add Up?","created":"2021-02-16","cited_by_count":18,"area":null},{"id":"2110.13148","title":"As if by magic: self-supervised training of deep despeckling networks with MERLIN","created":"2021-10-25","cited_by_count":18,"area":null},{"id":"2208.03113","title":"On the non-universality of deep learning: quantifying the cost of symmetry","created":"2022-08-05","cited_by_count":18,"area":null},{"id":"2110.01663","title":"Global Convergence and Stability of Stochastic Gradient Descent","created":"2021-10-04","cited_by_count":17,"area":null},{"id":"2303.09728","title":"The Cascaded Forward Algorithm for Neural Network Training","created":"2023-03-17","cited_by_count":17,"area":null},{"id":"2310.12474","title":"Enhancing High-Resolution 3D Generation through Pixel-wise Gradient Clipping","created":"2023-10-19","cited_by_count":17,"area":null},{"id":"2403.04081","title":"Directional Smoothness and Gradient Methods: Convergence and Adaptivity","created":"2024-03-06","cited_by_count":17,"area":null},{"id":"2506.01913","title":"Generalized Gradient Norm Clipping & Non-Euclidean $(L_0,L_1)$-Smoothness","created":"2025-06-02","cited_by_count":17,"area":null},{"id":"2204.02593","title":"Nonlinear gradient mappings and stochastic optimization: A general framework with applications to heavy-tail noise","created":"2022-04-06","cited_by_count":16,"area":null},{"id":"2206.00508","title":"Convergence of Stein Variational Gradient Descent under a Weaker Smoothness Condition","created":"2022-06-01","cited_by_count":16,"area":null},{"id":"2210.08343","title":"Modular machine learning-based elastoplasticity: generalization in the context of limited data","created":"2022-10-15","cited_by_count":16,"area":null},{"id":"2406.04592","title":"Provable Complexity Improvement of AdaGrad over SGD: Upper and Lower Bounds in Stochastic Non-Convex Optimization","created":"2024-06-07","cited_by_count":16,"area":null},{"id":"2110.06256","title":"Neural Network Weights Do Not Converge to Stationary Points: An Invariant Measure Perspective","created":"2021-10-12","cited_by_count":15,"area":null},{"id":"2401.09587","title":"Bilevel Optimization under Unbounded Smoothness: A New Algorithm and Convergence Analysis","created":"2024-01-17","cited_by_count":15,"area":null},{"id":"2402.13794","title":"Revisiting Convergence of AdaGrad with Relaxed Assumptions","created":"2024-02-21","cited_by_count":15,"area":null},{"id":"2409.01435","title":"Achieving Byzantine-Resilient Federated Learning via Layer-Adaptive Sparsified Model Aggregation","created":"2024-09-02","cited_by_count":15,"area":null},{"id":"2506.12543","title":"Is your batch size the problem? Revisiting the Adam-SGD gap in language modeling","created":"2025-06-14","cited_by_count":15,"area":null},{"id":"2105.01281","title":"Citadel: Protecting Data Privacy and Model Confidentiality for Collaborative Learning with SGX","created":"2021-05-04","cited_by_count":14,"area":null},{"id":"2107.04228","title":"Activated Gradients for Deep Neural Networks","created":"2021-07-09","cited_by_count":14,"area":null},{"id":"2110.11001","title":"Pixel-Level Face Image Quality Assessment for Explainable Face Recognition","created":"2021-10-21","cited_by_count":14,"area":null},{"id":"2206.01095","title":"Clipped Stochastic Methods for Variational Inequalities with Heavy-Tailed Noise","created":"2022-06-02","cited_by_count":14,"area":null},{"id":"2206.01367","title":"Adversarial Unlearning: Reducing Confidence Along Adversarial Directions","created":"2022-06-03","cited_by_count":14,"area":"safety"},{"id":"2304.05187","title":"Automatic Gradient Descent: Deep Learning without Hyperparameters","created":"2023-04-11","cited_by_count":14,"area":null},{"id":"2308.07770","title":"Multi-scale Promoted Self-adjusting Correlation Learning for Facial Action Unit Detection","created":"2023-08-15","cited_by_count":14,"area":null},{"id":"2403.15146","title":"On the Convergence of Adam under Non-uniform Smoothness: Separability from SGDM and Beyond","created":"2024-03-22","cited_by_count":14,"area":null},{"id":"2404.01436","title":"Convergence Guarantees for RMSProp and Adam in Generalized-smooth Non-convex Optimization with Affine Noise Variance","created":"2024-04-01","cited_by_count":14,"area":null},{"id":"2508.12531","title":"Rethinking Safety in LLM Fine-tuning: An Optimization Perspective","created":"2025-08-17","cited_by_count":14,"area":"lr"},{"id":"2206.00395","title":"Optimization with Access to Auxiliary Information","created":"2022-06-01","cited_by_count":13,"area":null},{"id":"2212.02083","title":"On the Overlooked Structure of Stochastic Gradients","created":"2022-12-05","cited_by_count":13,"area":null},{"id":"2302.09178","title":"Improving Training Stability for Multitask Ranking Models in Recommender Systems","created":"2023-02-17","cited_by_count":13,"area":null},{"id":"2502.07923","title":"Sign Operator for Coping with Heavy-Tailed Noise in Non-Convex Optimization: High Probability Bounds Under $(L_0, L_1)$-Smoothness","created":"2025-02-11","cited_by_count":13,"area":null},{"id":"2503.07978","title":"Detecting Backdoor Attacks in Federated Learning via Direction Alignment Inspection","created":"2025-03-11","cited_by_count":13,"area":null},{"id":"2505.05465","title":"ComPO: Preference Alignment via Comparison Oracles","created":"2025-05-08","cited_by_count":13,"area":"alignment"},{"id":"2506.16659","title":"Memory-Efficient LLM Pretraining via Minimalist Optimizer Design","created":"2025-06-20","cited_by_count":13,"area":null},{"id":"2104.09079","title":"A novel time-frequency Transformer based on self-attention mechanism and its application in fault diagnosis of rolling bearings","created":"2021-04-19","cited_by_count":12,"area":null},{"id":"2309.16398","title":"Recent Advances of Differential Privacy in Centralized Deep Learning: A Systematic Survey","created":"2023-09-28","cited_by_count":12,"area":null}],"next_cursor":"100"}