{"list":"other","total":99,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2201.11903","title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","created":"2022-01-28","cited_by_count":12459},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1608.03983","title":"SGDR: Stochastic Gradient Descent with Warm Restarts","created":"2016-08-13","cited_by_count":5045},{"id":"1804.07461","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","created":"2018-04-20","cited_by_count":4899},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2303.12712","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","created":"2023-03-22","cited_by_count":2592},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"1905.10044","title":"BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions","created":"2019-05-24","cited_by_count":2183},{"id":"2002.08909","title":"REALM: Retrieval-Augmented Language Model Pre-Training","created":"2020-02-10","cited_by_count":2141},{"id":"1808.06226","title":"SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing","created":"2018-08-19","cited_by_count":1989},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"1905.00537","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems","created":"2019-05-02","cited_by_count":1729},{"id":"2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","created":"2020-10-03","cited_by_count":1417},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"1506.06724","title":"Aligning Books and Movies: Towards Story-like Visual Explanations by Watching Movies and Reading Books","created":"2015-06-22","cited_by_count":1043},{"id":"1603.09382","title":"Deep Networks with Stochastic Depth","created":"2016-03-30","cited_by_count":979},{"id":"1804.04235","title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost","created":"2018-04-11","cited_by_count":905},{"id":"2204.07705","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","created":"2022-04-16","cited_by_count":868},{"id":"2009.06732","title":"Efficient Transformers: A Survey","created":"2020-09-14","cited_by_count":790},{"id":"1912.11370","title":"Big Transfer (BiT): General Visual Representation Learning","created":"2019-12-24","cited_by_count":769},{"id":"2107.06499","title":"Deduplicating Training Data Makes Language Models Better","created":"2021-07-14","cited_by_count":714},{"id":"2107.07075","title":"Deep Learning on a Data Diet: Finding Important Examples Early in Training","created":"2021-07-15","cited_by_count":533},{"id":"2206.14486","title":"Beyond neural scaling laws: beating power law scaling via data pruning","created":"2022-06-29","cited_by_count":528},{"id":"1911.11134","title":"Rigging the Lottery: Making All Tickets Winners","created":"2019-11-25","cited_by_count":495},{"id":"2104.14337","title":"Dynabench: Rethinking Benchmarking in NLP","created":"2021-04-07","cited_by_count":459},{"id":"1909.11556","title":"Reducing Transformer Depth on Demand with Structured Dropout","created":"2019-09-25","cited_by_count":439},{"id":"2302.06675","title":"Symbolic Discovery of Optimization Algorithms","created":"2023-02-13","cited_by_count":434},{"id":"2305.10429","title":"DoReMi: Optimizing Data Mixtures Speeds Up Language Model Pretraining","created":"2023-05-17","cited_by_count":399},{"id":"1906.11829","title":"Selection via Proxy: Efficient Data Selection for Deep Learning","created":"2019-06-26","cited_by_count":339},{"id":"1907.08610","title":"Lookahead Optimizer: k steps forward, 1 step back","created":"2019-07-19","cited_by_count":336},{"id":"2302.03169","title":"Data Selection for Language Models via Importance Resampling","created":"2023-02-06","cited_by_count":329},{"id":"1707.04780","title":"Scalable Training of Artificial Neural Networks with Adaptive Sparse Connectivity inspired by Network Science","created":"2017-07-15","cited_by_count":328},{"id":"1803.00942","title":"Not All Samples Are Created Equal: Deep Learning with Importance Sampling","created":"2018-03-02","cited_by_count":328},{"id":"1711.00489","title":"Don't Decay the Learning Rate, Increase the Batch Size","created":"2017-11-01","cited_by_count":320},{"id":"2303.09540","title":"SemDeDup: Data-efficient learning at web-scale through semantic deduplication","created":"2023-03-16","cited_by_count":303},{"id":"1802.06509","title":"On the Optimization of Deep Networks: Implicit Acceleration by Overparameterization","created":"2018-02-19","cited_by_count":284},{"id":"2010.07468","title":"AdaBelief Optimizer: Adapting Stepsizes by the Belief in Observed Gradients","created":"2020-10-15","cited_by_count":283},{"id":"2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","created":"2023-05-23","cited_by_count":265},{"id":"2003.04887","title":"ReZero is All You Need: Fast Convergence at Large Depth","created":"2020-03-10","cited_by_count":264},{"id":"2206.07137","title":"Prioritized Training on Points that are Learnable, Worth Learning, and Not Yet Learnt","created":"2022-06-14","cited_by_count":236},{"id":"1907.04840","title":"Sparse Networks from Scratch: Faster Training without Losing Performance","created":"2019-07-10","cited_by_count":213},{"id":"2204.05149","title":"The Carbon Footprint of Machine Learning Training Will Plateau, Then Shrink","created":"2022-04-11","cited_by_count":184},{"id":"2305.13169","title":"A Pretrainer's Guide to Training Data: Measuring the Effects of Data Age, Domain Coverage, Quality, & Toxicity","created":"2023-05-22","cited_by_count":182},{"id":"2207.07051","title":"Language models show human-like content effects on reasoning tasks","created":"2022-07-14","cited_by_count":173},{"id":"1907.04595","title":"Towards Explaining the Regularization Effect of Initial Large Learning Rate in Training Neural Networks","created":"2019-07-10","cited_by_count":162},{"id":"2106.08962","title":"Efficient Deep Learning: A Survey on Making Deep Learning Models Smaller, Faster, and Better","created":"2021-06-16","cited_by_count":154},{"id":"2204.08499","title":"DeepCore: A Comprehensive Library for Coreset Selection in Deep Learning","created":"2022-04-18","cited_by_count":147},{"id":"2209.06640","title":"Revisiting Neural Scaling Laws in Language and Vision","created":"2022-09-13","cited_by_count":132},{"id":"2007.01547","title":"Descending through a Crowded Valley - Benchmarking Deep Learning Optimizers","created":"2020-07-03","cited_by_count":119},{"id":"1802.06175","title":"An Alternative View: When Does SGD Escape Local Minima?","created":"2018-02-17","cited_by_count":115},{"id":"2012.03107","title":"When Do Curricula Work?","created":"2020-12-05","cited_by_count":106},{"id":"2207.10551","title":"Scaling Laws vs Model Architectures: How does Inductive Bias Influence Scaling?","created":"2022-07-21","cited_by_count":103},{"id":"2110.13711","title":"Hierarchical Transformers Are More Efficient Language Models","created":"2021-10-26","cited_by_count":102},{"id":"2110.12894","title":"The Efficiency Misnomer","created":"2021-10-25","cited_by_count":98},{"id":"1902.01996","title":"Are All Layers Created Equal?","created":"2019-02-06","cited_by_count":94},{"id":"1910.00762","title":"Accelerating Deep Learning by Focusing on the Biggest Losers","created":"2019-10-02","cited_by_count":94},{"id":"2010.13369","title":"Accelerating Training of Transformer-Based Language Models with Progressive Layer Dropping","created":"2020-10-26","cited_by_count":93},{"id":"2305.13230","title":"To Repeat or Not To Repeat: Insights from Scaling LLM under Token-Crisis","created":"2023-05-22","cited_by_count":92},{"id":"1902.00744","title":"Asymmetric Valleys: Beyond Sharp and Flat Local Minima","created":"2019-02-02","cited_by_count":90},{"id":"2102.11972","title":"Do Transformer Modifications Transfer Across Implementations and Applications?","created":"2021-02-23","cited_by_count":87},{"id":"2303.09752","title":"CoLT5: Faster Long-Range Transformers with Conditional Computation","created":"2023-03-17","cited_by_count":84},{"id":"2303.00980","title":"Learning to Grow Pretrained Models for Efficient Transformer Training","created":"2023-03-02","cited_by_count":83},{"id":"2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","created":"2022-12-28","cited_by_count":81},{"id":"2104.07705","title":"How to Train BERT with an Academic Budget","created":"2021-04-15","cited_by_count":80},{"id":"1511.06481","title":"Variance Reduction in SGD by Distributed Importance Sampling","created":"2015-11-20","cited_by_count":79},{"id":"1802.08770","title":"A Walk with SGD","created":"2018-02-24","cited_by_count":78},{"id":"2202.00661","title":"When Do Flat Minima Optimizers Work?","created":"2022-02-01","cited_by_count":78},{"id":"2211.09761","title":"Efficient Transformers with Dynamic Token Pooling","created":"2022-11-17","cited_by_count":76},{"id":"2304.08442","title":"The MiniPile Challenge for Data-Efficient Language Models","created":"2023-04-17","cited_by_count":75},{"id":"2210.05337","title":"SGD with Large Step Sizes Learns Sparse Features","created":"2022-10-11","cited_by_count":68},{"id":"1706.04983","title":"FreezeOut: Accelerate Training by Progressively Freezing Layers","created":"2017-06-15","cited_by_count":54},{"id":"2209.14981","title":"Stop Wasting My Time! Saving Days of ImageNet and BERT Training with Latest Weight Averaging","created":"2022-09-29","cited_by_count":54},{"id":"2306.07179","title":"Benchmarking Neural Network Training Algorithms","created":"2023-06-12","cited_by_count":52},{"id":"1907.04371","title":"Ordered SGD: A New Stochastic Optimization Framework for Empirical Risk Minimization","created":"2019-07-09","cited_by_count":51},{"id":"2203.06211","title":"Staged Training for Transformer Language Models","created":"2022-03-11","cited_by_count":49},{"id":"2102.01386","title":"AutoFreeze: Automatically Freezing Model Blocks to Accelerate Fine-tuning","created":"2021-02-02","cited_by_count":48},{"id":"2102.08098","title":"GradInit: Learning to Initialize Neural Networks for Stable and Efficient Training","created":"2021-02-16","cited_by_count":44},{"id":"1906.06669","title":"One Epoch Is All You Need","created":"2019-06-16","cited_by_count":41},{"id":"2302.01107","title":"A Survey on Efficient Training of Transformers","created":"2023-02-02","cited_by_count":39},{"id":"2210.06640","title":"Compute-Efficient Deep Learning: Algorithmic Trends and Opportunities","created":"2022-10-13","cited_by_count":38},{"id":"2203.14509","title":"Automated Progressive Learning for Efficient Training of Vision Transformers","created":"2022-03-28","cited_by_count":37},{"id":"2306.03241","title":"Early Weight Averaging meets High Learning Rates for LLM Pre-training","created":"2023-06-05","cited_by_count":37},{"id":"2111.04130","title":"NLP From Scratch Without Large-Scale Pretraining: A Simple and Efficient Framework","created":"2021-11-07","cited_by_count":34},{"id":"2304.03589","title":"On Efficient Training of Large-Scale Deep Learning Models: A Literature Review","created":"2023-04-07","cited_by_count":26},{"id":"2003.03977","title":"Wide-minima Density Hypothesis and the Explore-Exploit Learning Rate Schedule","created":"2020-03-09","cited_by_count":25},{"id":"2302.02596","title":"Ten Lessons We Have Learned in the New \"Sparseland\": A Short Handbook for Sparse Neural Network Researchers","created":"2023-02-06","cited_by_count":25},{"id":"2301.02312","title":"Training trajectories, mini-batch losses and the curious role of the learning rate","created":"2023-01-05","cited_by_count":21},{"id":"2211.05187","title":"Training a Vision Transformer from scratch in less than 24 hours with 1 GPU","created":"2022-11-09","cited_by_count":6},{"id":"2206.00832","title":"Fast Benchmarking of Accuracy vs. Training Time with Cyclic Learning Rates","created":"2022-06-02","cited_by_count":5}],"next_cursor":null}