{"list":"other","total":71,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1512.03385","title":"Deep Residual Learning for Image Recognition","created":"2015-12-10","cited_by_count":52096},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2105.05233","title":"Diffusion Models Beat GANs on Image Synthesis","created":"2021-05-11","cited_by_count":8255},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1607.06450","title":"Layer Normalization","created":"2016-07-21","cited_by_count":5669},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"2102.12092","title":"Zero-Shot Text-to-Image Generation","created":"2021-02-24","cited_by_count":4288},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"1609.07843","title":"Pointer Sentinel Mixture Models","created":"2016-09-26","cited_by_count":3096},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"1508.07909","title":"Neural Machine Translation of Rare Words with Subword Units","created":"2015-08-31","cited_by_count":2918},{"id":"1909.11942","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","created":"2019-09-26","cited_by_count":2790},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"1510.00149","title":"Deep Compression: Compressing Deep Neural Networks with Pruning, Trained Quantization and Huffman Coding","created":"2015-10-01","cited_by_count":2329},{"id":"2006.03654","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","created":"2020-06-05","cited_by_count":2181},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"1904.01038","title":"fairseq: A Fast, Extensible Toolkit for Sequence Modeling","created":"2019-04-01","cited_by_count":1488},{"id":"2112.11446","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","created":"2021-12-08","cited_by_count":1202},{"id":"2106.04560","title":"Scaling Vision Transformers","created":"2021-06-08","cited_by_count":999},{"id":"1603.05279","title":"XNOR-Net: ImageNet Classification Using Binary Convolutional Neural Networks","created":"2016-03-16","cited_by_count":849},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"2106.05974","title":"Scaling Vision with Sparse Mixture of Experts","created":"2021-06-10","cited_by_count":747},{"id":"1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","created":"2019-04-01","cited_by_count":674},{"id":"1511.00363","title":"BinaryConnect: Training Deep Neural Networks with binary weights during propagations","created":"2015-11-02","cited_by_count":592},{"id":"2106.04803","title":"CoAtNet: Marrying Convolution and Attention for All Data Sizes","created":"2021-06-09","cited_by_count":569},{"id":"2010.14701","title":"Scaling Laws for Autoregressive Generative Modeling","created":"2020-10-28","cited_by_count":522},{"id":"1705.09056","title":"Can Decentralized Algorithms Outperform Centralized Algorithms? A Case Study for Decentralized Parallel Stochastic Gradient Descent","created":"2017-05-25","cited_by_count":502},{"id":"1712.01887","title":"Deep Gradient Compression: Reducing the Communication Bandwidth for Distributed Training","created":"2017-12-05","cited_by_count":478},{"id":"2110.02861","title":"8-bit Optimizers via Block-wise Quantization","created":"2021-10-06","cited_by_count":403},{"id":"2106.04554","title":"A Survey of Transformers","created":"2021-06-08","cited_by_count":401},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"1404.5997","title":"One weird trick for parallelizing convolutional neural networks","created":"2014-04-23","cited_by_count":264},{"id":"2107.02137","title":"ERNIE 3.0: Large-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-07-05","cited_by_count":260},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"1905.13727","title":"PowerSGD: Practical Low-Rank Gradient Compression for Distributed Optimization","created":"2019-05-31","cited_by_count":238},{"id":"1809.10853","title":"Adaptive Input Representations for Neural Language Modeling","created":"2018-09-28","cited_by_count":226},{"id":"1806.00187","title":"Scaling Neural Machine Translation","created":"2018-06-01","cited_by_count":204},{"id":"2004.03333","title":"Binary Neural Networks: A Survey","created":"2020-03-31","cited_by_count":190},{"id":"1302.4389","title":"Maxout Networks","created":"2013-02-18","cited_by_count":187},{"id":"1807.05358","title":"Beyond Data and Model Parallelism for Deep Neural Networks","created":"2018-07-14","cited_by_count":179},{"id":"1811.02084","title":"Mesh-TensorFlow: Deep Learning for Supercomputers","created":"2018-11-05","cited_by_count":169},{"id":"1802.09941","title":"Demystifying Parallel and Distributed Deep Learning: An In-Depth Concurrency Analysis","created":"2018-02-26","cited_by_count":156},{"id":"1907.05242","title":"Large Memory Layers with Product Keys","created":"2019-07-10","cited_by_count":124},{"id":"2206.01288","title":"Decentralized Training of Foundation Models in Heterogeneous Environments","created":"2022-06-02","cited_by_count":87},{"id":"2102.02888","title":"1-bit Adam: Communication Efficient Large-Scale Training with Adam's Convergence Speed","created":"2021-02-04","cited_by_count":75},{"id":"2106.10207","title":"Distributed Deep Learning in Open Collaborations","created":"2021-06-18","cited_by_count":58},{"id":"1511.04561","title":"8-Bit Approximations for Parallelism in Deep Learning","created":"2015-11-14","cited_by_count":56},{"id":"2010.05846","title":"A Refined Laser Method and Faster Matrix Multiplication","created":"2020-10-12","cited_by_count":56},{"id":"1910.05124","title":"PipeMare: Asynchronous Pipeline Parallel DNN Training","created":"2019-10-09","cited_by_count":55},{"id":"2204.12013","title":"Bamboo: Making Preemptible Instances Resilient for Affordable Training of Large DNNs","created":"2022-04-26","cited_by_count":49},{"id":"1806.10188","title":"A Tight Convergence Analysis for Stochastic Gradient Descent with Delayed Updates","created":"2018-06-26","cited_by_count":47},{"id":"2002.04013","title":"Towards Crowdsourced Training of Large Neural Networks using Decentralized Mixture-of-Experts","created":"2020-02-10","cited_by_count":43},{"id":"2103.03239","title":"Moshpit SGD: Communication-Efficient Decentralized Training on Heterogeneous Unreliable Devices","created":"2021-03-04","cited_by_count":36},{"id":"2002.05645","title":"Training Large Neural Networks with Constant Memory using a New Execution Algorithm","created":"2020-02-13","cited_by_count":34},{"id":"2206.01299","title":"Fine-tuning Language Models over Slow Networks using Activation Compression with Guarantees","created":"2022-06-02","cited_by_count":30},{"id":"2003.06307","title":"Communication-Efficient Distributed Deep Learning: A Comprehensive Survey","created":"2020-03-10","cited_by_count":26},{"id":"2104.06069","title":"1-bit LAMB: Communication Efficient Large-Scale Large-Batch Training with LAMB's Convergence Speed","created":"2021-04-13","cited_by_count":24},{"id":"2005.00124","title":"Breaking (Global) Barriers in Parallel Stochastic Optimization with Wait-Avoiding Group Averaging","created":"2020-04-30","cited_by_count":9},{"id":"2103.08894","title":"Distributed Deep Learning Using Volunteer Computing-Like Paradigm","created":"2021-03-16","cited_by_count":9},{"id":"2008.00177","title":"Multi-node Bert-pretraining: Cost-efficient Approach","created":"2020-08-01","cited_by_count":5},{"id":"2003.08372","title":"Interleaved Weighted Round-Robin: A Network Calculus Analysis","created":"2020-03-18","cited_by_count":4},{"id":"1903.00045","title":"Speeding up Deep Learning with Transient Servers","created":"2019-02-28","cited_by_count":3},{"id":"2003.05649","title":"Machine Learning on Volatile Instances","created":"2020-03-12","cited_by_count":3},{"id":"1906.03496","title":"Making Asynchronous Stochastic Gradient Descent Work for Transformers","created":"2019-06-08","cited_by_count":2}],"next_cursor":null}