{"list":"other","total":81,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"2103.00020","title":"Learning Transferable Visual Models From Natural Language Supervision","created":"2021-02-26","cited_by_count":33744},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2106.09685","title":"LoRA: Low-Rank Adaptation of Large Language Models","created":"2021-06-17","cited_by_count":14380},{"id":"2111.06377","title":"Masked Autoencoders Are Scalable Vision Learners","created":"2021-11-11","cited_by_count":7403},{"id":"2012.12877","title":"Training data-efficient image transformers & distillation through attention","created":"2020-12-23","cited_by_count":4586},{"id":"1902.00751","title":"Parameter-Efficient Transfer Learning for NLP","created":"2019-02-02","cited_by_count":4236},{"id":"2104.08691","title":"The Power of Scale for Parameter-Efficient Prompt Tuning","created":"2021-04-18","cited_by_count":3795},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1909.11942","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","created":"2019-09-26","cited_by_count":2790},{"id":"2106.08254","title":"BEiT: BERT Pre-Training of Image Transformers","created":"2021-06-15","cited_by_count":2375},{"id":"1803.03635","title":"The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks","created":"2018-03-09","cited_by_count":2262},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2203.12119","title":"Visual Prompt Tuning","created":"2022-03-23","cited_by_count":1681},{"id":"1706.02677","title":"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour","created":"2017-06-08","cited_by_count":1492},{"id":"2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","created":"2020-10-03","cited_by_count":1417},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"1906.02243","title":"Energy and Policy Considerations for Deep Learning in NLP","created":"2019-06-05","cited_by_count":1254},{"id":"2106.10199","title":"BitFit: Simple Parameter-efficient Fine-tuning for Transformer-based Masked Language-models","created":"2021-06-18","cited_by_count":1245},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"1606.04838","title":"Optimization Methods for Large-Scale Machine Learning","created":"2016-06-15","cited_by_count":1150},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"1810.02340","title":"SNIP: Single-shot Network Pruning based on Connection Sensitivity","created":"2018-10-04","cited_by_count":812},{"id":"1811.03962","title":"A Convergence Theory for Deep Learning via Over-Parameterization","created":"2018-11-09","cited_by_count":787},{"id":"2103.17239","title":"Going deeper with Image Transformers","created":"2021-03-31","cited_by_count":764},{"id":"1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","created":"2019-04-01","cited_by_count":674},{"id":"2012.13255","title":"Intrinsic Dimensionality Explains the Effectiveness of Language Model Fine-Tuning","created":"2020-12-22","cited_by_count":671},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"1907.10597","title":"Green AI","created":"2019-07-22","cited_by_count":627},{"id":"1704.04760","title":"In-Datacenter Performance Analysis of a Tensor Processing Unit","created":"2017-04-16","cited_by_count":618},{"id":"1606.06160","title":"DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients","created":"2016-06-20","cited_by_count":569},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"1708.03888","title":"Large Batch Training of Convolutional Networks","created":"2017-08-13","cited_by_count":535},{"id":"2107.07075","title":"Deep Learning on a Data Diet: Finding Important Examples Early in Training","created":"2021-07-15","cited_by_count":533},{"id":"2206.14486","title":"Beyond neural scaling laws: beating power law scaling via data pruning","created":"2022-06-29","cited_by_count":528},{"id":"1911.11134","title":"Rigging the Lottery: Making All Tickets Winners","created":"2019-11-25","cited_by_count":495},{"id":"2103.10697","title":"ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases","created":"2021-03-19","cited_by_count":484},{"id":"2002.07376","title":"Picking Winning Tickets Before Training by Preserving Gradient Flow","created":"2020-02-18","cited_by_count":471},{"id":"1609.07061","title":"Quantized Neural Networks: Training Neural Networks with Low Precision Weights and Activations","created":"2016-09-22","cited_by_count":426},{"id":"1905.02450","title":"MASS: Masked Sequence to Sequence Pre-training for Language Generation","created":"2019-05-07","cited_by_count":347},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"1811.04918","title":"Learning and Generalization in Overparameterized Neural Networks, Going Beyond Two Layers","created":"2018-11-12","cited_by_count":340},{"id":"1803.00942","title":"Not All Samples Are Created Equal: Deep Learning with Importance Sampling","created":"2018-03-02","cited_by_count":328},{"id":"2212.00794","title":"Scaling Language-Image Pre-training via Masking","created":"2022-12-01","cited_by_count":317},{"id":"1911.03584","title":"On the Relationship between Self-Attention and Convolutional Layers","created":"2019-11-08","cited_by_count":300},{"id":"1802.06509","title":"On the Optimization of Deep Networks: Implicit Acceleration by Overparameterization","created":"2018-02-19","cited_by_count":284},{"id":"2007.10319","title":"MCUNet: Tiny Deep Learning on IoT Devices","created":"2020-07-20","cited_by_count":273},{"id":"2003.04887","title":"ReZero is All You Need: Fast Convergence at Large Depth","created":"2020-03-10","cited_by_count":264},{"id":"1808.01204","title":"Learning Overparameterized Neural Networks via Stochastic Gradient Descent on Structured Data","created":"2018-08-03","cited_by_count":261},{"id":"2106.01548","title":"When Vision Transformers Outperform ResNets without Pre-training or Strong Data Augmentations","created":"2021-06-03","cited_by_count":260},{"id":"2104.08378","title":"Accelerating Sparse Deep Neural Networks","created":"2021-04-16","cited_by_count":223},{"id":"1901.09321","title":"Fixup Initialization: Residual Learning Without Normalization","created":"2019-01-27","cited_by_count":198},{"id":"2107.00651","title":"AutoFormer: Searching Transformers for Visual Recognition","created":"2021-07-01","cited_by_count":188},{"id":"2106.04533","title":"Chasing Sparsity in Vision Transformers: An End-to-End Exploration","created":"2021-06-08","cited_by_count":183},{"id":"1901.11117","title":"The Evolved Transformer","created":"2019-01-30","cited_by_count":180},{"id":"1812.08011","title":"Training Deep Neural Networks with 8-bit Floating Point Numbers","created":"2018-12-19","cited_by_count":164},{"id":"1909.11957","title":"Drawing Early-Bird Tickets: Towards More Efficient Training of Deep Networks","created":"2019-09-26","cited_by_count":160},{"id":"2110.03141","title":"Efficient Sharpness-aware Minimization for Improved Training of Neural Networks","created":"2021-10-07","cited_by_count":137},{"id":"2002.10941","title":"A$^3$: Accelerating Attention Mechanisms in Neural Networks with Approximation","created":"2020-02-22","cited_by_count":101},{"id":"2110.14883","title":"Colossal-AI: A Unified Deep Learning System For Large-Scale Parallel Training","created":"2021-10-28","cited_by_count":96},{"id":"2205.14083","title":"Sharpness-Aware Training for Free","created":"2022-05-27","cited_by_count":95},{"id":"2010.13369","title":"Accelerating Training of Transformer-Based Language Models with Progressive Layer Dropping","created":"2020-10-26","cited_by_count":93},{"id":"2101.00063","title":"EarlyBERT: Efficient BERT Training via Early-bird Lottery Tickets","created":"2020-12-31","cited_by_count":79},{"id":"2002.11794","title":"Train Large, Then Compress: Rethinking Model Size for Efficient Training and Inference of Transformers","created":"2020-02-26","cited_by_count":67},{"id":"2104.14129","title":"ActNN: Reducing Training Memory Footprint via 2-Bit Activation Compressed Training","created":"2021-04-29","cited_by_count":61},{"id":"2209.14981","title":"Stop Wasting My Time! Saving Days of ImageNet and BERT Training with Latest Weight Averaging","created":"2022-09-29","cited_by_count":54},{"id":"2010.12562","title":"On the Transformer Growth for Progressive BERT Training","created":"2020-10-23","cited_by_count":49},{"id":"1802.00930","title":"Mixed Precision Training of Convolutional Neural Networks using Integer Operations","created":"2018-02-03","cited_by_count":45},{"id":"2210.09573","title":"ViTCoD: Vision Transformer Acceleration via Dedicated Algorithm and Accelerator Co-Design","created":"2022-10-18","cited_by_count":41},{"id":"2203.14509","title":"Automated Progressive Learning for Efficient Training of Vision Transformers","created":"2022-03-28","cited_by_count":37},{"id":"1912.03194","title":"Why are Adaptive Methods Good for Attention Models?","created":"2019-12-06","cited_by_count":35},{"id":"2206.11357","title":"GACT: Activation Compressed Training for Generic Network Architectures","created":"2022-06-22","cited_by_count":31},{"id":"2004.09910","title":"torchgpipe: On-the-fly Pipeline Parallelism for Training Giant Models","created":"2020-04-21","cited_by_count":26},{"id":"2001.05674","title":"Shifted and Squeezed 8-bit Floating Point format for Low-Precision Training of Deep Neural Networks","created":"2020-01-16","cited_by_count":23},{"id":"1911.13214","title":"Optimal checkpointing for heterogeneous chains: how to train deep neural networks with limited memory","created":"2019-11-27","cited_by_count":22},{"id":"1705.04925","title":"Convergence Analysis of Proximal Gradient with Momentum for Nonconvex Optimization","created":"2017-05-14","cited_by_count":18},{"id":"2111.11124","title":"Mesa: A Memory-saving Training Framework for Transformers","created":"2021-11-22","cited_by_count":16}],"next_cursor":null}