{"list":"other","total":51,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1607.06450","title":"Layer Normalization","created":"2016-07-21","cited_by_count":5669},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"2211.05100","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","created":"2022-11-09","cited_by_count":1984},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"1706.02677","title":"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour","created":"2017-06-08","cited_by_count":1492},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"2212.07143","title":"Reproducible scaling laws for contrastive language-image learning","created":"2022-12-14","cited_by_count":1173},{"id":"2106.04560","title":"Scaling Vision Transformers","created":"2021-06-08","cited_by_count":999},{"id":"2404.06395","title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","created":"2024-04-09","cited_by_count":723},{"id":"2204.06745","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","created":"2022-04-14","cited_by_count":713},{"id":"2302.05442","title":"Scaling Vision Transformers to 22 Billion Parameters","created":"2023-02-10","cited_by_count":694},{"id":"2401.02954","title":"DeepSeek LLM: Scaling Open-Source Language Models with Longtermism","created":"2024-01-05","cited_by_count":642},{"id":"1712.00409","title":"Deep Learning Scaling is Predictable, Empirically","created":"2017-12-01","cited_by_count":582},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","created":"2023-04-28","cited_by_count":523},{"id":"2010.14701","title":"Scaling Laws for Autoregressive Generative Modeling","created":"2020-10-28","cited_by_count":522},{"id":"2305.16264","title":"Scaling Data-Constrained Language Models","created":"2023-05-25","cited_by_count":403},{"id":"1812.06162","title":"An Empirical Model of Large-Batch Training","created":"2018-12-14","cited_by_count":281},{"id":"1909.12673","title":"A Constructive Prediction of the Generalization Error Across Scales","created":"2019-09-27","cited_by_count":210},{"id":"1811.03600","title":"Measuring the Effects of Data Parallelism on Neural Network Training","created":"2018-11-08","cited_by_count":193},{"id":"2309.14322","title":"Small-scale proxies for large-scale Transformer training instabilities","created":"2023-09-25","cited_by_count":183},{"id":"2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","created":"2024-05-28","cited_by_count":170},{"id":"2401.00448","title":"Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws","created":"2023-12-31","cited_by_count":159},{"id":"1907.04164","title":"Which Algorithmic Choices Matter at Which Batch Sizes? Insights From a Noisy Quadratic Model","created":"2019-07-09","cited_by_count":117},{"id":"2210.16859","title":"A Solvable Model of Neural Scaling Laws","created":"2022-10-30","cited_by_count":116},{"id":"2208.09632","title":"Adam Can Converge Without Any Modification On Update Rules","created":"2022-08-20","cited_by_count":114},{"id":"2403.08540","title":"Language models scale reliably with over-training and on downstream tasks","created":"2024-03-13","cited_by_count":113},{"id":"2402.01092","title":"A Dynamical Model of Neural Scaling Laws","created":"2024-02-02","cited_by_count":111},{"id":"2109.10686","title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","created":"2021-09-22","cited_by_count":106},{"id":"2402.17834","title":"Stable LM 2 1.6B Technical Report","created":"2024-02-27","cited_by_count":90},{"id":"2305.13035","title":"Getting ViT in Shape: Scaling Laws for Compute-Optimal Model Design","created":"2023-05-22","cited_by_count":89},{"id":"2404.07177","title":"Scaling Laws for Data Filtering -- Data Curation cannot be Compute Agnostic","created":"2024-04-10","cited_by_count":77},{"id":"2404.10102","title":"Chinchilla Scaling: A replication attempt","created":"2024-04-15","cited_by_count":72},{"id":"2405.15074","title":"4+3 Phases of Compute-Optimal Neural Scaling Laws","created":"2024-05-23","cited_by_count":68},{"id":"1908.11365","title":"Improving Deep Transformer with Depth-Scaled Initialization and Merged Attention","created":"2019-08-29","cited_by_count":59},{"id":"2406.08466","title":"Scaling Laws in Linear Regression: Compute, Parameters, and Data","created":"2024-06-12","cited_by_count":57},{"id":"2406.12907","title":"Reconciling Kaplan and Chinchilla Scaling Laws","created":"2024-06-12","cited_by_count":39},{"id":"2306.13575","title":"Scaling MLPs: A Tale of Inductive Bias","created":"2023-06-23","cited_by_count":37},{"id":"2212.12147","title":"The Onset of Variance-Limited Behavior for Networks in the Lazy and Rich Regimes","created":"2022-12-23","cited_by_count":25},{"id":"2202.06387","title":"Scaling Laws Under the Microscope: Predicting Transformer Performance from Small Scale Experiments","created":"2022-02-13","cited_by_count":23}],"next_cursor":null}