{"list":"other","total":74,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1512.03385","title":"Deep Residual Learning for Image Recognition","created":"2015-12-10","cited_by_count":52096},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"1409.1556","title":"Very Deep Convolutional Networks for Large-Scale Image Recognition","created":"2014-09-04","cited_by_count":15364},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2312.00752","title":"Mamba: Linear-Time Sequence Modeling with Selective State Spaces","created":"2023-12-01","cited_by_count":4718},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"2105.01601","title":"MLP-Mixer: An all-MLP Architecture for Vision","created":"2021-05-04","cited_by_count":1446},{"id":"2002.04745","title":"On Layer Normalization in the Transformer Architecture","created":"2020-02-12","cited_by_count":922},{"id":"2210.02414","title":"GLM-130B: An Open Bilingual Pre-trained Model","created":"2022-10-05","cited_by_count":843},{"id":"1904.09237","title":"On the Convergence of Adam and Beyond","created":"2019-04-19","cited_by_count":841},{"id":"1503.05671","title":"Optimizing Neural Networks with Kronecker-factored Approximate Curvature","created":"2015-03-19","cited_by_count":796},{"id":"1908.03265","title":"On the Variance of the Adaptive Learning Rate and Beyond","created":"2019-08-08","cited_by_count":772},{"id":"2302.05442","title":"Scaling Vision Transformers to 22 Billion Parameters","created":"2023-02-10","cited_by_count":694},{"id":"2309.10305","title":"Baichuan 2: Open Large-scale Language Models","created":"2023-09-19","cited_by_count":670},{"id":"1802.04434","title":"signSGD: Compressed Optimisation for Non-Convex Problems","created":"2018-02-13","cited_by_count":571},{"id":"2103.03404","title":"Attention is Not All You Need: Pure Attention Loses Rank Doubly Exponentially with Depth","created":"2021-03-05","cited_by_count":419},{"id":"2106.14881","title":"Early Convolutions Help Transformers See Better","created":"2021-06-28","cited_by_count":396},{"id":"1611.01838","title":"Entropy-SGD: Biasing Gradient Descent Into Wide Valleys","created":"2016-11-06","cited_by_count":359},{"id":"2305.17333","title":"Fine-Tuning Language Models with Just Forward Passes","created":"2023-05-27","cited_by_count":345},{"id":"1905.11881","title":"Why gradient clipping accelerates training: A theoretical justification for adaptivity","created":"2019-05-28","cited_by_count":341},{"id":"1906.01787","title":"Learning Deep Transformer Models for Machine Translation","created":"2019-06-05","cited_by_count":309},{"id":"1901.10159","title":"An Investigation into Neural Net Optimization via Hessian Eigenvalue Density","created":"2019-01-29","cited_by_count":292},{"id":"1912.07145","title":"PyHessian: Neural Networks Through the Lens of the Hessian","created":"2019-12-16","cited_by_count":292},{"id":"1706.04454","title":"Empirical Analysis of the Hessian of Over-Parametrized Neural Networks","created":"2017-06-14","cited_by_count":290},{"id":"2203.03466","title":"Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer","created":"2022-03-07","cited_by_count":288},{"id":"2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","created":"2023-05-23","cited_by_count":265},{"id":"2003.04887","title":"ReZero is All You Need: Fast Convergence at Large Depth","created":"2020-03-10","cited_by_count":264},{"id":"1812.04754","title":"Gradient Descent Happens in a Tiny Subspace","created":"2018-12-12","cited_by_count":217},{"id":"2004.08249","title":"Understanding the Difficulty of Training Transformers","created":"2020-04-17","cited_by_count":210},{"id":"1902.09843","title":"Adaptive Gradient Methods with Dynamic Bound of Learning Rate","created":"2019-02-26","cited_by_count":183},{"id":"2309.14322","title":"Small-scale proxies for large-scale Transformer training instabilities","created":"2023-09-25","cited_by_count":183},{"id":"2203.00555","title":"DeepNet: Scaling Transformers to 1,000 Layers","created":"2022-03-01","cited_by_count":178},{"id":"1808.02941","title":"On the Convergence of A Class of Adam-Type Algorithms for Non-Convex Optimization","created":"2018-08-08","cited_by_count":174},{"id":"2003.02395","title":"A Simple Convergence Proof of Adam and Adagrad","created":"2020-03-05","cited_by_count":171},{"id":"2303.06296","title":"Stabilizing Transformer Training by Preventing Attention Entropy Collapse","created":"2023-03-11","cited_by_count":161},{"id":"1611.07476","title":"Eigenvalues of the Hessian in Deep Learning: Singularity and Beyond","created":"2016-11-22","cited_by_count":159},{"id":"1910.05895","title":"Transformers without Tears: Improving the Normalization of Self-Attention","created":"2019-10-14","cited_by_count":153},{"id":"2206.03126","title":"Signal Propagation in Transformers: Theoretical Perspectives and the Role of Rank Collapse","created":"2022-06-07","cited_by_count":141},{"id":"2306.09782","title":"Full Parameter Fine-tuning for Large Language Models with Limited Resources","created":"2023-06-16","cited_by_count":134},{"id":"1811.09358","title":"A Sufficient Condition for Convergences of Adam and RMSProp","created":"2018-11-23","cited_by_count":126},{"id":"1907.04164","title":"Which Algorithmic Choices Matter at Which Batch Sizes? Insights From a Noisy Quadratic Model","created":"2019-07-09","cited_by_count":117},{"id":"2208.09632","title":"Adam Can Converge Without Any Modification On Update Rules","created":"2022-08-20","cited_by_count":114},{"id":"2304.13960","title":"Noise Is Not the Main Factor Behind the Gap Between SGD and Adam on Transformers, but Sign Descent Might Be","created":"2023-04-27","cited_by_count":114},{"id":"1804.09849","title":"The Best of Both Worlds: Combining Recent Advances in Neural Machine Translation","created":"2018-04-26","cited_by_count":102},{"id":"2304.13972","title":"Convergence of Adam Under Relaxed Assumptions","created":"2023-04-27","cited_by_count":99},{"id":"2310.01082","title":"Linear attention is (maybe) all you need (to understand transformer optimization)","created":"2023-10-02","cited_by_count":98},{"id":"2008.11865","title":"Traces of Class/Cross-Class Structure Pervade Deep Learning Spectra","created":"2020-08-27","cited_by_count":94},{"id":"1808.05671","title":"On the Convergence of Adaptive Gradient Methods for Nonconvex Optimization","created":"2018-08-16","cited_by_count":91},{"id":"1802.08241","title":"Hessian-based Analysis of Large Batch Training and Robustness to Adversaries","created":"2018-02-22","cited_by_count":87},{"id":"2208.11195","title":"Robustness to Unbounded Smoothness of Generalized SignSGD","created":"2022-08-23","cited_by_count":81},{"id":"1912.08957","title":"Optimization for deep learning: theory and algorithms","created":"2019-12-19","cited_by_count":70},{"id":"1901.08244","title":"Measurements of Three-Level Hierarchical Structure in the Outliers in the Spectrum of Deepnet Hessians","created":"2019-01-24","cited_by_count":68},{"id":"1908.11365","title":"Improving Deep Transformer with Depth-Scaled Initialization and Merged Attention","created":"2019-08-29","cited_by_count":59},{"id":"2305.18471","title":"Convergence of AdaGrad for Non-convex Objectives: Simple Proofs and Relaxed Assumptions","created":"2023-05-29","cited_by_count":57},{"id":"2304.09871","title":"A Theory on Adam Instability in Large-Scale Machine Learning","created":"2023-04-19","cited_by_count":56},{"id":"2306.00204","title":"Toward Understanding Why Adam Converges Faster Than SGD for Transformers","created":"2023-05-31","cited_by_count":53},{"id":"2010.04261","title":"Dissecting Hessian: Understanding Common Structure of Hessian in Neural Networks","created":"2020-10-08","cited_by_count":45},{"id":"2012.03801","title":"A Deeper Look at the Hessian Eigenspectrum of Deep Neural Networks and its Applications to Regularization","created":"2020-12-07","cited_by_count":44},{"id":"2006.15815","title":"Adaptive Inertia: Disentangling the Effects of Adaptive Learning Rate and Momentum","created":"2020-06-29","cited_by_count":40},{"id":"2010.09697","title":"Effects of Parameter Norm Growth During Transformer Training: Inductive Bias from Gradient Descent","created":"2020-10-19","cited_by_count":40},{"id":"2310.17998","title":"Closing the Gap Between the Upper Bound and the Lower Bound of Adam's Iteration Complexity","created":"2023-10-27","cited_by_count":38},{"id":"2103.01519","title":"Hessian Eigenspectra of More Realistic Nonlinear Models","created":"2021-03-02","cited_by_count":33},{"id":"1810.13108","title":"A general system of differential equations to model first order adaptive algorithms","created":"2018-10-31","cited_by_count":28},{"id":"2101.05471","title":"Towards Practical Adam: Non-Convexity, Convergence Theory, and Mini-Batch Acceleration","created":"2021-01-14","cited_by_count":25},{"id":"2211.02254","title":"How Does Adaptive Optimization Impact Local Neural Network Geometry?","created":"2022-11-04","cited_by_count":24},{"id":"1802.03451","title":"Estimating the Spectral Density of Large Implicit Matrices","created":"2018-02-09","cited_by_count":16},{"id":"2012.05640","title":"Asymptotic study of stochastic adaptive algorithm in non-convex landscape","created":"2020-12-10","cited_by_count":16},{"id":"2106.09687","title":"Super-Acceleration with Cyclical Step-sizes","created":"2021-06-17","cited_by_count":14},{"id":"2210.02070","title":"Non-Convergence and Limit Cycles in the Adam optimizer","created":"2022-10-05","cited_by_count":9}],"next_cursor":null}