{"total":87,"rows":[{"id":"2406.16793","title":"Adam-mini: Use Fewer Learning Rates To Gain More","created":"2024-06-24","cited_by_count":137,"area":null},{"id":"2402.16788","title":"Why Transformers Need Adam: A Hessian Perspective","created":"2024-02-26","cited_by_count":124,"area":null},{"id":"2402.19449","title":"Heavy-Tailed Class Imbalance and Why Adam Outperforms Gradient Descent on Language Models","created":"2024-02-29","cited_by_count":80,"area":null},{"id":"2410.08292","title":"Can Looped Transformers Learn to Implement Multi-step Gradient Descent for In-context Learning?","created":"2024-10-10","cited_by_count":65,"area":null},{"id":"2312.06528","title":"Transformers Implement Functional Gradient Descent to Learn Non-Linear Functions In Context","created":"2023-12-11","cited_by_count":63,"area":null},{"id":"2410.13835","title":"Active-Dormant Attention Heads: Mechanistically Demystifying Extreme-Token Phenomena in LLMs","created":"2024-10-17","cited_by_count":56,"area":null},{"id":"2410.13849","title":"From Gradient Clipping to Normalization for Heavy Tailed SGD","created":"2024-10-17","cited_by_count":49,"area":null},{"id":"2402.01258","title":"Transformers Learn Nonlinear Features In Context: Nonconvex Mean-field Dynamics on the Attention Landscape","created":"2024-02-02","cited_by_count":46,"area":null},{"id":"2501.16265","title":"Training Dynamics of In-Context Learning in Linear Attention","created":"2025-01-27","cited_by_count":46,"area":null},{"id":"2405.19592","title":"Why Larger Language Models Do In-context Learning Differently?","created":"2024-05-30","cited_by_count":45,"area":null},{"id":"2402.10198","title":"SAMformer: Unlocking the Potential of Transformers in Time Series Forecasting with Sharpness-Aware Minimization and Channel-Wise Attention","created":"2024-02-15","cited_by_count":44,"area":null},{"id":"2407.07972","title":"Deconstructing What Makes a Good Optimizer for Language Models","created":"2024-07-10","cited_by_count":41,"area":null},{"id":"2506.04192","title":"Lions and Muons: Optimization via Stochastic Frank-Wolfe under Heavy-Tailed Noise","created":"2025-06-04","cited_by_count":41,"area":null},{"id":"2405.16002","title":"Does SGD really happen in tiny subspaces?","created":"2024-05-25","cited_by_count":36,"area":null},{"id":"2411.02544","title":"Pretrained transformer efficiently learns low-dimensional target functions in-context","created":"2024-11-04","cited_by_count":31,"area":null},{"id":"2405.16411","title":"Training Tensor Attention Efficiently: From Cubic to Almost Linear Time","created":"2024-05-26","cited_by_count":30,"area":null},{"id":"2402.01567","title":"Understanding Adam Optimizer via Online Learning of Updates: Adam is FTRL in Disguise","created":"2024-02-02","cited_by_count":26,"area":null},{"id":"2405.18199","title":"Adam with model exponential moving average is effective for nonconvex optimization","created":"2024-05-28","cited_by_count":23,"area":null},{"id":"2401.04301","title":"Setting the Record Straight on Transformer Oversmoothing","created":"2024-01-09","cited_by_count":21,"area":null},{"id":"2401.17426","title":"Superiority of Multi-Head Attention in In-Context Linear Regression","created":"2024-01-30","cited_by_count":17,"area":null},{"id":"2508.04860","title":"Can SGD Handle Heavy-Tailed Noise?","created":"2025-08-06","cited_by_count":17,"area":null},{"id":"2506.12543","title":"Is your batch size the problem? Revisiting the Adam-SGD gap in language modeling","created":"2025-06-14","cited_by_count":15,"area":null},{"id":"2402.02212","title":"A Survey to Recent Progress Towards Understanding In-Context Learning","created":"2024-02-03","cited_by_count":14,"area":null},{"id":"2402.05173","title":"Towards Understanding Inductive Bias in Transformers: A View From Infinity","created":"2024-02-07","cited_by_count":13,"area":null},{"id":"2410.21698","title":"On the Role of Depth and Looping for In-Context Learning with Task Diversity","created":"2024-10-29","cited_by_count":11,"area":null},{"id":"2410.10986","title":"What Does It Mean to Be a Transformer? Insights from a Theoretical Hessian Analysis","created":"2024-10-14","cited_by_count":9,"area":null},{"id":"2502.21009","title":"Position: Solve Layerwise Linear Models First to Understand Neural Dynamical Phenomena (Neural Collapse, Emergence, Lazy/Rich Regime, and Grokking)","created":"2025-02-28","cited_by_count":9,"area":null},{"id":"2505.11892","title":"Fast RoPE Attention: Combining the Polynomial Method and Fast Fourier Transform","created":"2025-05-17","cited_by_count":9,"area":null},{"id":"2402.11215","title":"AdAdaGrad: Adaptive Batch Size Schemes for Adaptive Gradient Methods","created":"2024-02-17","cited_by_count":8,"area":null},{"id":"2605.06884","title":"Muon with Nesterov Momentum: Heavy-Tailed Noise and (Randomized) Inexact Polar Decomposition","created":"2026-05-07","cited_by_count":8,"area":null},{"id":"2501.18187","title":"On the Role of Transformer Feed-Forward Layers in Nonlinear In-Context Learning","created":"2025-01-30","cited_by_count":7,"area":null},{"id":"2502.04164","title":"Efficient Distributed Optimization under Heavy-Tailed Noise","created":"2025-02-06","cited_by_count":7,"area":null},{"id":"2505.16284","title":"Only Large Weights (And Not Skip Connections) Can Prevent the Perils of Rank Collapse","created":"2025-05-22","cited_by_count":7,"area":null},{"id":"2602.15322","title":"On Surprising Effectiveness of Masking Updates in Adaptive Optimizers","created":"2026-02-17","cited_by_count":7,"area":null},{"id":"2403.11963","title":"Transfer Learning Beyond Bounded Density Ratios","created":"2024-03-18","cited_by_count":6,"area":null},{"id":"2409.12293","title":"In-Context Learning of Linear Systems: Generalization Theory and Applications to Operator Learning","created":"2024-09-18","cited_by_count":6,"area":null},{"id":"2505.14808","title":"Out-of-Distribution Generalization of In-Context Learning: A Low-Dimensional Subspace Perspective","created":"2025-05-20","cited_by_count":6,"area":null},{"id":"2511.21622","title":"On the Origin of Algorithmic Progress in AI","created":"2025-11-26","cited_by_count":6,"area":null},{"id":"2402.00743","title":"Theoretical Understanding of In-Context Learning in Shallow Transformers with Unstructured Data","created":"2024-02-01","cited_by_count":5,"area":null},{"id":"2405.16845","title":"On Mesa-Optimization in Autoregressively Trained Transformers: Emergence and Capability","created":"2024-05-27","cited_by_count":5,"area":null},{"id":"2410.10101","title":"Learning Linear Attention in Polynomial Time","created":"2024-10-14","cited_by_count":5,"area":null},{"id":"2603.01514","title":"Training Dynamics of Softmax Self-Attention: Fast Global Convergence via Preconditioning","created":"2026-03-02","cited_by_count":5,"area":null},{"id":"2505.17852","title":"Scaling Recurrent Neural Networks to a Billion Parameters with Zero-Order Optimization","created":"2025-05-23","cited_by_count":4,"area":null},{"id":"2506.11769","title":"Long-Short Alignment for Effective Long-Context Modeling in LLMs","created":"2025-06-13","cited_by_count":4,"area":null},{"id":"2509.14952","title":"Stochastic Bilevel Optimization with Heavy-Tailed Noise","created":"2025-09-18","cited_by_count":4,"area":null},{"id":"2405.17248","title":"On Understanding Attention-Based In-Context Learning for Categorical Data","created":"2024-05-27","cited_by_count":3,"area":null},{"id":"2502.07553","title":"Transformers Provably Learn Sparse XOR with Polylogarithmic Parameters","created":"2025-02-11","cited_by_count":3,"area":null},{"id":"2503.01843","title":"When Can You Get Away with Low Memory Adam?","created":"2025-03-03","cited_by_count":3,"area":null},{"id":"2508.07571","title":"Towards Theoretical Understanding of Transformer Test-Time Computing: Investigation on In-Context Linear Regression","created":"2025-08-11","cited_by_count":3,"area":null},{"id":"2510.10690","title":"Second-order Optimization under Heavy-Tailed Noise: Hessian Clipping and Sample Complexity Limits","created":"2025-10-12","cited_by_count":3,"area":null},{"id":"2510.22421","title":"Extragradient Method for $(L_0, L_1)$-Lipschitz Root-finding Problems","created":"2025-10-25","cited_by_count":3,"area":null},{"id":"2512.11784","title":"Softmax as Linear Attention in the Large-Prompt Regime: a Measure-based Perspective","created":"2025-12-12","cited_by_count":3,"area":null},{"id":"2605.18694","title":"Can Adaptive Gradient Methods Converge under Heavy-Tailed Noise? A Case Study of AdaGrad","created":"2026-05-18","cited_by_count":3,"area":null},{"id":"2606.00520","title":"In-Expectation Convergence of Stochastic Gradient Methods under Heavy-Tailed Noise","created":"2026-05-30","cited_by_count":3,"area":null},{"id":"2408.02288","title":"Spin glass model of in-context learning","created":"2024-08-05","cited_by_count":2,"area":null},{"id":"2510.13680","title":"Adam or Gauss-Newton? A Comparative Study In Terms of Basis Alignment and SGD Noise","created":"2025-10-15","cited_by_count":2,"area":null},{"id":"2510.26303","title":"Implicit Bias of Per-sample Adam on Separable Data: Departure from the Full-batch Regime","created":"2025-10-30","cited_by_count":2,"area":null},{"id":"2511.04456","title":"Federated Stochastic Minimax Optimization under Heavy-Tailed Noises","created":"2025-11-06","cited_by_count":2,"area":null},{"id":"2601.19379","title":"Optimal Asynchronous Stochastic Nonconvex Optimization under Heavy-Tailed Noise","created":"2026-01-27","cited_by_count":2,"area":null},{"id":"2602.13413","title":"Why is Normalization Preferred? A Worst-Case Complexity Theory for Stochastically Preconditioned SGD under Heavy-Tailed Noise","created":"2026-02-13","cited_by_count":2,"area":null},{"id":"2603.15059","title":"Muon Converges under Heavy-Tailed Noise: Nonconvex H\\\"{o}lder-Smooth Empirical Risk Minimization","created":"2026-03-16","cited_by_count":2,"area":null},{"id":"2605.11838","title":"Gradient Clipping Beyond Vector Norms: A Spectral Approach for Matrix-Valued Parameters","created":"2026-05-12","cited_by_count":2,"area":null},{"id":"2606.14560","title":"Free Heavy-Tailed Lunch for Muon: A Theoretical Justification of Empirical Success","created":"2026-06-12","cited_by_count":2,"area":null},{"id":"2607.27383","title":"The Convergence Behavior of Adam under Heavy-Tailed Noise","created":"2026-07-29","cited_by_count":2,"area":null},{"id":"2609.08659","title":"TriCCOT: Tri-part Convolutional Conformal Transformer for Onboard Space Object Detection","created":"2026-09-08","cited_by_count":2,"area":null},{"id":"2410.24050","title":"A Mechanistic Study of Transformers Training Dynamics","created":"2024-10-31","cited_by_count":1,"area":null},{"id":"2503.03908","title":"On the Convergence of Adam-Type Algorithm for Bilevel Optimization under Unbounded Smoothness","created":"2025-03-05","cited_by_count":1,"area":null},{"id":"2505.15009","title":"How Transformers Learn In-Context Recall Tasks? Optimality, Training Dynamics and Generalization","created":"2025-05-21","cited_by_count":1,"area":null},{"id":"2509.15543","title":"Nonconvex Decentralized Stochastic Bilevel Optimization under Heavy-Tailed Noise","created":"2025-09-19","cited_by_count":1,"area":null},{"id":"2601.11435","title":"Near-Optimal Decentralized Stochastic Nonconvex Optimization with Heavy-Tailed Noise","created":"2026-01-16","cited_by_count":1,"area":null},{"id":"2602.02707","title":"Every Bit Counts: A Theoretical Study of Precision-Expressivity Tradeoffs in Quantized Transformers","created":"2026-02-02","cited_by_count":1,"area":null},{"id":"2602.23197","title":"Fine-Tuning Without Forgetting In-Context Learning: A Theoretical Analysis of Linear Attention Models","created":"2026-02-26","cited_by_count":1,"area":null},{"id":"2603.22155","title":"RAMPAGE: RAndomized Mid-Point for debiAsed Gradient Extrapolation","created":"2026-03-23","cited_by_count":1,"area":null},{"id":"2605.15787","title":"Grokking as Structural Inference: Transformers Need Bayesian Lottery Tickets","created":"2026-05-15","cited_by_count":1,"area":null},{"id":"2606.06814","title":"The Effect of Training Task Diversity on In-Context Learning through the Lens of Low-Dimensional Subspaces","created":"2026-06-05","cited_by_count":1,"area":null},{"id":"2606.14259","title":"Beyond a Single Explanation of the Adam--SGD Gap","created":"2026-06-12","cited_by_count":1,"area":null},{"id":"2606.23676","title":"Open Problem: Is AdamW Effective Under Heavy-Tailed Noise?","created":"2026-06-22","cited_by_count":1,"area":null},{"id":"2607.08104","title":"Vanilla SGD with Momentum Survives Heavy-Tailed Noise: Convergence Analysis without Gradient Clipping or Normalization","created":"2026-07-09","cited_by_count":1,"area":null},{"id":"2502.04580","title":"Technical Debt in In-Context Learning: Diminishing Efficiency in Long Context","created":"2025-02-07","cited_by_count":0,"area":null},{"id":"2511.03270","title":"SCALE: Upscaled Continual Learning of Large Language Models","created":"2025-11-05","cited_by_count":0,"area":null},{"id":"2603.13085","title":"Linearized Attention Cannot Enter the Kernel Regime at Any Practical Width","created":"2026-03-13","cited_by_count":0,"area":null},{"id":"2604.01995","title":"MTLSI-Net: A Linear Semantic Interaction Network for Parameter-Efficient Multi-Task Dense Prediction","created":"2026-04-02","cited_by_count":0,"area":null},{"id":"2605.11416","title":"Freeze Deep, Train Shallow: Interpretable Layer Allocation for Continued Pre-Training","created":"2026-05-12","cited_by_count":0,"area":null},{"id":"2605.24513","title":"Zeroth-Order Nonconvex Nonsmooth Optimization with Heavy-Tailed Noise","created":"2026-05-23","cited_by_count":0,"area":null},{"id":"2609.30732","title":"TR-SSQP: A Trust-Region Method for Constrained Stochastic Optimization under Heavy-Tailed Noise","created":"2026-09-25","cited_by_count":0,"area":null},{"id":"2609.37659","title":"Are In-Context Images Worth 10 Dimensions?","created":"2026-09-29","cited_by_count":0,"area":null},{"id":"2610.17061","title":"In-context classification: Calibration and noise in linear attention with softmax readout","created":"2026-10-12","cited_by_count":0,"area":null}],"next_cursor":null}