{"list":"other","total":107,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"1907.11692","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","created":"2019-07-26","cited_by_count":13372},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"1804.07461","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","created":"2018-04-20","cited_by_count":4899},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2004.05150","title":"Longformer: The Long-Document Transformer","created":"2020-04-10","cited_by_count":3413},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1508.07909","title":"Neural Machine Translation of Rare Words with Subword Units","created":"2015-08-31","cited_by_count":2918},{"id":"1906.08237","title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding","created":"2019-06-19","cited_by_count":2811},{"id":"1909.11942","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","created":"2019-09-26","cited_by_count":2790},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"1910.03771","title":"HuggingFace's Transformers: State-of-the-art Natural Language Processing","created":"2019-10-09","cited_by_count":2218},{"id":"1808.06226","title":"SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing","created":"2018-08-19","cited_by_count":1989},{"id":"1905.00537","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems","created":"2019-05-02","cited_by_count":1729},{"id":"2007.14062","title":"Big Bird: Transformers for Longer Sequences","created":"2020-07-28","cited_by_count":1684},{"id":"1910.07467","title":"Root Mean Square Layer Normalization","created":"2019-10-16","cited_by_count":1480},{"id":"1609.08144","title":"Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation","created":"2016-09-26","cited_by_count":1475},{"id":"2006.04768","title":"Linformer: Self-Attention with Linear Complexity","created":"2020-06-08","cited_by_count":1474},{"id":"2002.10957","title":"MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers","created":"2020-02-25","cited_by_count":1463},{"id":"2111.09543","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing","created":"2021-11-18","cited_by_count":1254},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"1909.10351","title":"TinyBERT: Distilling BERT for Natural Language Understanding","created":"2019-09-23","cited_by_count":1167},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"1906.02629","title":"When Does Label Smoothing Help?","created":"2019-06-06","cited_by_count":988},{"id":"1805.12471","title":"Neural Network Acceptability Judgments","created":"2018-05-31","cited_by_count":958},{"id":"2002.04745","title":"On Layer Normalization in the Transformer Architecture","created":"2020-02-12","cited_by_count":922},{"id":"1804.04235","title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost","created":"2018-04-11","cited_by_count":905},{"id":"2112.03570","title":"Membership Inference Attacks From First Principles","created":"2021-12-07","cited_by_count":878},{"id":"1907.10529","title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans","created":"2019-07-24","cited_by_count":834},{"id":"2009.06732","title":"Efficient Transformers: A Survey","created":"2020-09-14","cited_by_count":790},{"id":"1908.03265","title":"On the Variance of the Adaptive Learning Rate and Beyond","created":"2019-08-08","cited_by_count":772},{"id":"2107.06499","title":"Deduplicating Training Data Makes Language Models Better","created":"2021-07-14","cited_by_count":714},{"id":"2204.06745","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","created":"2022-04-14","cited_by_count":713},{"id":"1612.08083","title":"Language Modeling with Gated Convolutional Networks","created":"2016-12-23","cited_by_count":702},{"id":"2011.04006","title":"Long Range Arena: A Benchmark for Efficient Transformers","created":"2020-11-08","cited_by_count":700},{"id":"1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","created":"2019-04-01","cited_by_count":674},{"id":"1804.10959","title":"Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates","created":"2018-04-29","cited_by_count":639},{"id":"1912.02178","title":"Fantastic Generalization Measures and Where to Find Them","created":"2019-12-04","cited_by_count":513},{"id":"1908.09355","title":"Patient Knowledge Distillation for BERT Model Compression","created":"2019-08-25","cited_by_count":508},{"id":"2004.02984","title":"MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices","created":"2020-04-06","cited_by_count":482},{"id":"2102.06701","title":"Explaining Neural Scaling Laws","created":"2021-02-12","cited_by_count":363},{"id":"2105.03824","title":"FNet: Mixing Tokens with Fourier Transforms","created":"2021-05-09","cited_by_count":347},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2202.01169","title":"Unified Scaling Laws for Routed Language Models","created":"2022-02-02","cited_by_count":253},{"id":"2206.07137","title":"Prioritized Training on Points that are Learnable, Worth Learning, and Not Yet Learnt","created":"2022-06-14","cited_by_count":236},{"id":"1809.10853","title":"Adaptive Input Representations for Neural Language Modeling","created":"2018-09-28","cited_by_count":226},{"id":"2004.08249","title":"Understanding the Difficulty of Training Transformers","created":"2020-04-17","cited_by_count":210},{"id":"2002.12804","title":"UniLMv2: Pseudo-Masked Language Models for Unified Language Model Pre-Training","created":"2020-02-28","cited_by_count":204},{"id":"2006.15595","title":"Rethinking Positional Encoding in Language Pre-training","created":"2020-06-28","cited_by_count":197},{"id":"2202.10447","title":"Transformer Quality in Linear Time","created":"2022-02-21","cited_by_count":191},{"id":"2202.00622","title":"Datamodels: Predicting Predictions from Training Data","created":"2022-02-01","cited_by_count":189},{"id":"2109.08668","title":"Primer: Searching for Efficient Transformers for Language Modeling","created":"2021-09-17","cited_by_count":164},{"id":"2009.02276","title":"Witches' Brew: Industrial Scale Data Poisoning via Gradient Matching","created":"2020-09-04","cited_by_count":163},{"id":"2006.03236","title":"Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing","created":"2020-06-05","cited_by_count":162},{"id":"2009.06489","title":"The Hardware Lottery","created":"2020-09-14","cited_by_count":160},{"id":"2205.10487","title":"Scaling Laws and Interpretability of Learning from Repeated Data","created":"2022-05-21","cited_by_count":152},{"id":"2204.05832","title":"What Language Model Architecture and Pretraining Objective Work Best for Zero-Shot Generalization?","created":"2022-04-12","cited_by_count":150},{"id":"1905.07799","title":"Adaptive Attention Span in Transformers","created":"2019-05-19","cited_by_count":149},{"id":"2006.07322","title":"Evaluation of Neural Architectures Trained with Square Loss vs Cross-Entropy in Classification Tasks","created":"2020-06-12","cited_by_count":138},{"id":"2202.08005","title":"Should You Mask 15% in Masked Language Modeling?","created":"2022-02-16","cited_by_count":118},{"id":"2110.08529","title":"Sharpness-Aware Minimization Improves Language Model Generalization","created":"2021-10-16","cited_by_count":108},{"id":"2010.12821","title":"Rethinking embedding coupling in pre-trained language models","created":"2020-10-24","cited_by_count":103},{"id":"2207.10551","title":"Scaling Laws vs Model Architectures: How does Inductive Bias Influence Scaling?","created":"2022-07-21","cited_by_count":103},{"id":"2110.13711","title":"Hierarchical Transformers Are More Efficient Language Models","created":"2021-10-26","cited_by_count":102},{"id":"2110.12894","title":"The Efficiency Misnomer","created":"2021-10-25","cited_by_count":98},{"id":"2010.13369","title":"Accelerating Training of Transformer-Based Language Models with Progressive Layer Dropping","created":"2020-10-26","cited_by_count":93},{"id":"2106.16163","title":"The MultiBERTs: BERT Reproductions for Robustness Analysis","created":"2021-06-30","cited_by_count":92},{"id":"2210.15424","title":"What Language Model to Train if You Have One Million GPU Hours?","created":"2022-10-27","cited_by_count":92},{"id":"2102.11972","title":"Do Transformer Modifications Transfer Across Implementations and Applications?","created":"2021-02-23","cited_by_count":87},{"id":"2110.09456","title":"NormFormer: Improved Transformer Pretraining with Extra Normalization","created":"2021-10-18","cited_by_count":83},{"id":"2104.07705","title":"How to Train BERT with an Academic Budget","created":"2021-04-15","cited_by_count":80},{"id":"2209.00099","title":"Efficient Methods for Natural Language Processing: A Survey","created":"2022-08-31","cited_by_count":78},{"id":"2104.07885","title":"Probing Across Time: What Does RoBERTa Know and When?","created":"2021-04-16","cited_by_count":73},{"id":"2202.01994","title":"Data Scaling Laws in NMT: The Effect of Noise and Architecture","created":"2022-02-04","cited_by_count":55},{"id":"2012.08561","title":"Pre-Training Transformers as Energy-Based Cloze Models","created":"2020-12-15","cited_by_count":52},{"id":"1901.01672","title":"Generalization in Deep Networks: The Role of Distance from Initialization","created":"2019-01-07","cited_by_count":51},{"id":"2006.11316","title":"SqueezeBERT: What can computer vision teach NLP about efficient neural networks?","created":"2020-06-19","cited_by_count":50},{"id":"2010.12562","title":"On the Transformer Growth for Progressive BERT Training","created":"2020-10-23","cited_by_count":49},{"id":"2203.06211","title":"Staged Training for Transformer Language Models","created":"2022-03-11","cited_by_count":49},{"id":"1802.05374","title":"A Progressive Batching L-BFGS Method for Machine Learning","created":"2018-02-15","cited_by_count":45},{"id":"2102.08098","title":"GradInit: Learning to Initialize Neural Networks for Stable and Efficient Training","created":"2021-02-16","cited_by_count":44},{"id":"2203.13240","title":"Token Dropping for Efficient BERT Pretraining","created":"2022-03-24","cited_by_count":42},{"id":"1906.06669","title":"One Epoch Is All You Need","created":"2019-06-16","cited_by_count":41},{"id":"1710.11258","title":"Adaptive Sampling Strategies for Stochastic Optimization","created":"2017-10-30","cited_by_count":30},{"id":"1911.11423","title":"Single Headed Attention RNN: Stop Thinking With Your Head","created":"2019-11-26","cited_by_count":26},{"id":"2011.02266","title":"Optimizing Transformer for Low-Resource Neural Machine Translation","created":"2020-11-04","cited_by_count":24},{"id":"2005.09561","title":"Normalized Attention Without Probability Cage","created":"2020-05-19","cited_by_count":20},{"id":"2204.06644","title":"METRO: Efficient Denoising Pretraining of Large Scale Autoencoding Language Models with Model Generated Signals","created":"2022-04-13","cited_by_count":20},{"id":"2202.04350","title":"pNLP-Mixer: an Efficient all-MLP Architecture for Language","created":"2022-02-09","cited_by_count":19},{"id":"2106.04563","title":"XtremeDistilTransformers: Task Transfer for Task-agnostic Distillation","created":"2021-06-08","cited_by_count":18},{"id":"2105.03928","title":"Which transformer architecture fits my data? A vocabulary bottleneck in self-attention","created":"2021-05-09","cited_by_count":16},{"id":"2106.09650","title":"Multi-head or Single-head? An Empirical Comparison for Transformer Training","created":"2021-06-17","cited_by_count":16},{"id":"2205.11726","title":"On the Role of Bidirectionality in Language Model Pre-Training","created":"2022-05-24","cited_by_count":15}],"next_cursor":"100"}