{"list":"other","total":67,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2103.00020","title":"Learning Transferable Visual Models From Natural Language Supervision","created":"2021-02-26","cited_by_count":33744},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2005.12872","title":"End-to-End Object Detection with Transformers","created":"2020-05-26","cited_by_count":7414},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2012.07805","title":"Extracting Training Data from Large Language Models","created":"2020-12-14","cited_by_count":2189},{"id":"2002.08909","title":"REALM: Retrieval-Augmented Language Model Pre-Training","created":"2020-02-10","cited_by_count":2141},{"id":"2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","created":"2022-06-09","cited_by_count":1998},{"id":"2005.08100","title":"Conformer: Convolution-augmented Transformer for Speech Recognition","created":"2020-05-16","cited_by_count":1863},{"id":"1905.00537","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems","created":"2019-05-02","cited_by_count":1729},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"2105.01601","title":"MLP-Mixer: An all-MLP Architecture for Vision","created":"2021-05-04","cited_by_count":1446},{"id":"2112.04426","title":"Improving language models by retrieving from trillions of tokens","created":"2021-12-08","cited_by_count":1293},{"id":"2111.09883","title":"Swin Transformer V2: Scaling Up Capacity and Resolution","created":"2021-11-18","cited_by_count":1228},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897},{"id":"1911.00172","title":"Generalization through Memorization: Nearest Neighbor Language Models","created":"2019-11-01","cited_by_count":800},{"id":"2009.06732","title":"Efficient Transformers: A Survey","created":"2020-09-14","cited_by_count":790},{"id":"2202.09368","title":"Mixture-of-Experts with Expert Choice Routing","created":"2022-02-18","cited_by_count":623},{"id":"2104.10350","title":"Carbon Emissions and Large Neural Network Training","created":"2021-04-21","cited_by_count":567},{"id":"2010.11125","title":"Beyond English-Centric Multilingual Machine Translation","created":"2020-10-21","cited_by_count":561},{"id":"1911.11134","title":"Rigging the Lottery: Making All Tickets Winners","created":"2019-11-25","cited_by_count":495},{"id":"1902.09574","title":"The State of Sparsity in Deep Neural Networks","created":"2019-02-25","cited_by_count":476},{"id":"2201.05596","title":"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale","created":"2022-01-14","cited_by_count":404},{"id":"1611.06194","title":"Expert Gate: Lifelong Learning with a Network of Experts","created":"2016-11-18","cited_by_count":372},{"id":"2103.16716","title":"BASE Layers: Simplifying Training of Large, Sparse Models","created":"2021-03-30","cited_by_count":320},{"id":"2207.07061","title":"Confident Adaptive Language Modeling","created":"2022-07-14","cited_by_count":310},{"id":"1312.3005","title":"One Billion Word Benchmark for Measuring Progress in Statistical Language Modeling","created":"2013-12-11","cited_by_count":308},{"id":"1312.4314","title":"Learning Factored Representations in a Deep Mixture of Experts","created":"2013-12-16","cited_by_count":278},{"id":"2202.01169","title":"Unified Scaling Laws for Routed Language Models","created":"2022-02-02","cited_by_count":253},{"id":"2106.04426","title":"Hash Layers For Large Sparse Models","created":"2021-06-08","cited_by_count":237},{"id":"1907.04840","title":"Sparse Networks from Scratch: Faster Training without Losing Performance","created":"2019-07-10","cited_by_count":213},{"id":"2206.02770","title":"Multimodal Contrastive Learning with LIMoE: the Language-Image Mixture of Experts","created":"2022-06-06","cited_by_count":203},{"id":"2206.03382","title":"Tutel: Adaptive Mixture-of-Experts at Scale","created":"2022-06-07","cited_by_count":201},{"id":"2208.03306","title":"Branch-Train-Merge: Embarrassingly Parallel Training of Expert Language Models","created":"2022-08-05","cited_by_count":191},{"id":"2203.17189","title":"Scaling Up Models and Data with $\\texttt{t5x}$ and $\\texttt{seqio}$","created":"2022-03-31","cited_by_count":187},{"id":"2112.10684","title":"Efficient Large Scale Language Modeling with Mixtures of Experts","created":"2021-12-20","cited_by_count":175},{"id":"1811.02084","title":"Mesh-TensorFlow: Deep Learning for Supercomputers","created":"2018-11-05","cited_by_count":169},{"id":"1511.06297","title":"Conditional Computation in Neural Networks for faster models","created":"2015-11-19","cited_by_count":167},{"id":"2108.05036","title":"DEMix Layers: Disentangling Domains for Modular Language Modeling","created":"2021-08-11","cited_by_count":132},{"id":"1905.06874","title":"Behavior Sequence Transformer for E-commerce Recommendation in Alibaba","created":"2019-05-15","cited_by_count":130},{"id":"1711.01239","title":"Routing Networks: Adaptive Selection of Non-linear Functions for Multi-Task Learning","created":"2017-11-03","cited_by_count":126},{"id":"1806.03377","title":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","created":"2018-06-08","cited_by_count":117},{"id":"2106.03760","title":"DSelect-k: Differentiable Selection in the Mixture of Experts with Applications to Multi-Task Learning","created":"2021-06-07","cited_by_count":115},{"id":"2110.04260","title":"Taming Sparsely Activated Transformer with Stochastic Experts","created":"2021-10-08","cited_by_count":113},{"id":"2110.03742","title":"Beyond Distillation: Task-level Mixture-of-Experts for Efficient Inference","created":"2021-09-24","cited_by_count":96},{"id":"2102.11972","title":"Do Transformer Modifications Transfer Across Implementations and Applications?","created":"2021-02-23","cited_by_count":87},{"id":"2111.12763","title":"Sparse is Enough in Scaling Transformers","created":"2021-11-24","cited_by_count":86},{"id":"2109.10465","title":"Scalable and Efficient MoE Training for Multitask Multilingual Models","created":"2021-09-22","cited_by_count":76},{"id":"1811.05249","title":"Modular Networks: Learning to Decompose Neural Computation","created":"2018-11-13","cited_by_count":62},{"id":"1904.12774","title":"Routing Networks and the Challenges of Modular and Compositional Computation","created":"2019-04-29","cited_by_count":50},{"id":"2009.13239","title":"Scalable Transfer Learning with Expert Models","created":"2020-09-28","cited_by_count":49},{"id":"2105.03036","title":"SpeechMoE: Scaling to Large Acoustic Models with Dynamic Routing Mixture of Experts","created":"2021-05-07","cited_by_count":48},{"id":"2111.11831","title":"SpeechMoE2: Mixture-of-Experts Model with Improved Routing","created":"2021-11-23","cited_by_count":26},{"id":"2202.07101","title":"A Survey on Dynamic Neural Networks for Natural Language Processing","created":"2022-02-15","cited_by_count":24},{"id":"2106.09563","title":"On Anytime Learning at Macroscale","created":"2021-06-17","cited_by_count":20},{"id":"2110.08246","title":"Tricks for Training Sparse Translation Models","created":"2021-10-15","cited_by_count":17},{"id":"2112.05820","title":"Building a great multi-lingual teacher with sparsely-gated mixture of experts for speech recognition","created":"2021-12-10","cited_by_count":16},{"id":"2205.12399","title":"Sparse Mixers: Combining MoE and Mixing to build a more efficient BERT","created":"2022-05-24","cited_by_count":12},{"id":"2205.01848","title":"Optimizing Mixture of Experts using Dynamic Recompilations","created":"2022-05-04","cited_by_count":4}],"next_cursor":null}