{"list":"other","total":54,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"1607.06450","title":"Layer Normalization","created":"2016-07-21","cited_by_count":5669},{"id":"1608.03983","title":"SGDR: Stochastic Gradient Descent with Warm Restarts","created":"2016-08-13","cited_by_count":5045},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"1905.07830","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","created":"2019-05-19","cited_by_count":4100},{"id":"1606.08415","title":"Gaussian Error Linear Units (GELUs)","created":"2016-06-27","cited_by_count":3502},{"id":"1609.07843","title":"Pointer Sentinel Mixture Models","created":"2016-09-26","cited_by_count":3096},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1911.11641","title":"PIQA: Reasoning about Physical Commonsense in Natural Language","created":"2019-11-26","cited_by_count":2883},{"id":"1905.10044","title":"BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions","created":"2019-05-24","cited_by_count":2183},{"id":"1808.06226","title":"SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing","created":"2018-08-19","cited_by_count":1989},{"id":"1603.05027","title":"Identity Mappings in Deep Residual Networks","created":"2016-03-16","cited_by_count":1965},{"id":"1901.02860","title":"Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context","created":"2019-01-09","cited_by_count":1850},{"id":"1904.10509","title":"Generating Long Sequences with Sparse Transformers","created":"2019-04-23","cited_by_count":1546},{"id":"1910.07467","title":"Root Mean Square Layer Normalization","created":"2019-10-16","cited_by_count":1480},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"1803.02155","title":"Self-Attention with Relative Position Representations","created":"2018-03-06","cited_by_count":1120},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103},{"id":"2002.04745","title":"On Layer Normalization in the Transformer Architecture","created":"2020-02-12","cited_by_count":922},{"id":"1503.05671","title":"Optimizing Neural Networks with Kronecker-factored Approximate Curvature","created":"2015-03-19","cited_by_count":796},{"id":"2103.17239","title":"Going deeper with Image Transformers","created":"2021-03-31","cited_by_count":764},{"id":"1711.00165","title":"Deep Neural Networks as Gaussian Processes","created":"2017-11-01","cited_by_count":622},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"1802.09568","title":"Shampoo: Preconditioned Stochastic Tensor Optimization","created":"2018-02-26","cited_by_count":426},{"id":"2103.03404","title":"Attention is Not All You Need: Pure Attention Loses Rank Doubly Exponentially with Depth","created":"2021-03-05","cited_by_count":419},{"id":"1804.11271","title":"Gaussian Process Behaviour in Wide Deep Neural Networks","created":"2018-04-30","cited_by_count":317},{"id":"1906.01787","title":"Learning Deep Transformer Models for Machine Translation","created":"2019-06-05","cited_by_count":309},{"id":"2003.04887","title":"ReZero is All You Need: Fast Convergence at Large Depth","created":"2020-03-10","cited_by_count":264},{"id":"1606.05340","title":"Exponential expressivity in deep neural networks through transient chaos","created":"2016-06-16","cited_by_count":261},{"id":"1809.10853","title":"Adaptive Input Representations for Neural Language Modeling","created":"2018-09-28","cited_by_count":226},{"id":"2004.08249","title":"Understanding the Difficulty of Training Transformers","created":"2020-04-17","cited_by_count":210},{"id":"1901.09321","title":"Fixup Initialization: Residual Learning Without Normalization","created":"2019-01-27","cited_by_count":198},{"id":"1611.01232","title":"Deep Information Propagation","created":"2016-11-04","cited_by_count":195},{"id":"2203.00555","title":"DeepNet: Scaling Transformers to 1,000 Layers","created":"2022-03-01","cited_by_count":178},{"id":"1806.05393","title":"Dynamical Isometry and a Mean Field Theory of CNNs: How to Train 10,000-Layer Vanilla Convolutional Neural Networks","created":"2018-06-14","cited_by_count":176},{"id":"1702.08591","title":"The Shattered Gradients Problem: If resnets are the answer, then what is the question?","created":"2017-02-28","cited_by_count":157},{"id":"1910.12478","title":"Tensor Programs I: Wide Feedforward or Recurrent Neural Networks of Any Architecture are Gaussian Processes","created":"2019-10-28","cited_by_count":151},{"id":"2206.03126","title":"Signal Propagation in Transformers: Theoretical Perspectives and the Role of Rank Collapse","created":"2022-06-07","cited_by_count":141},{"id":"1602.05897","title":"Toward Deeper Understanding of Neural Networks: The Power of Initialization and a Dual View on Expressivity","created":"2016-02-18","cited_by_count":138},{"id":"2206.00133","title":"Pre-training via Denoising for Molecular Property Prediction","created":"2022-05-31","cited_by_count":124},{"id":"2006.10540","title":"Infinite attention: NNGP and NTK for deep attention networks","created":"2020-06-18","cited_by_count":121},{"id":"1803.01719","title":"How to Start Training: The Effect of Initialization and Architecture","created":"2018-03-05","cited_by_count":109},{"id":"1902.06853","title":"On the Impact of the Activation Function on Deep Neural Networks Training","created":"2019-02-19","cited_by_count":102},{"id":"1908.11365","title":"Improving Deep Transformer with Depth-Scaled Initialization and Merged Attention","created":"2019-08-29","cited_by_count":59},{"id":"2110.01765","title":"Rapid training of deep neural networks without skip connections or normalization layers using Deep Kernel Shaping","created":"2021-10-05","cited_by_count":52},{"id":"2010.12859","title":"Stable ResNet","created":"2020-10-24","cited_by_count":49},{"id":"2206.02768","title":"The Neural Covariance SDE: Shaped Infinite Depth-and-Width Networks at Initialization","created":"2022-06-06","cited_by_count":41},{"id":"1906.02341","title":"How to Initialize your Network? Robust Initialization for WeightNorm & ResNets","created":"2019-06-05","cited_by_count":36},{"id":"2203.08120","title":"Deep Learning without Shortcuts: Shaping the Kernel with Tailored Rectifiers","created":"2022-03-15","cited_by_count":28},{"id":"1911.03179","title":"Lipschitz Constrained Parameter Initialization for Deep Transformers","created":"2019-11-08","cited_by_count":9},{"id":"2104.05878","title":"On the validity of kernel approximations for orthogonally-initialized neural networks","created":"2021-04-13","cited_by_count":4}],"next_cursor":null}