{"list":"other","total":56,"rows":[{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"2106.09685","title":"LoRA: Low-Rank Adaptation of Large Language Models","created":"2021-06-17","cited_by_count":14380},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2307.15818","title":"RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control","created":"2023-07-28","cited_by_count":3080},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"2210.17323","title":"GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers","created":"2022-10-31","cited_by_count":1919},{"id":"2401.04088","title":"Mixtral of Experts","created":"2024-01-08","cited_by_count":1823},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"2301.00774","title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","created":"2023-01-02","cited_by_count":1121},{"id":"2112.06905","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","created":"2021-12-13","cited_by_count":897},{"id":"1905.09418","title":"Analyzing Multi-Head Self-Attention: Specialized Heads Do the Heavy Lifting, the Rest Can Be Pruned","created":"2019-05-23","cited_by_count":856},{"id":"2306.11695","title":"A Simple and Effective Pruning Approach for Large Language Models","created":"2023-06-20","cited_by_count":800},{"id":"2306.01116","title":"The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only","created":"2023-06-01","cited_by_count":763},{"id":"2305.11627","title":"LLM-Pruner: On the Structural Pruning of Large Language Models","created":"2023-05-19","cited_by_count":650},{"id":"2202.09368","title":"Mixture-of-Experts with Expert Choice Routing","created":"2022-02-18","cited_by_count":623},{"id":"2104.10350","title":"Carbon Emissions and Large Neural Network Training","created":"2021-04-21","cited_by_count":567},{"id":"2102.00554","title":"Sparsity in Deep Learning: Pruning and growth for efficient inference and training in neural networks","created":"2021-01-31","cited_by_count":527},{"id":"1908.09355","title":"Patient Knowledge Distillation for BERT Model Compression","created":"2019-08-25","cited_by_count":508},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2201.05596","title":"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale","created":"2022-01-14","cited_by_count":404},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399},{"id":"2103.16716","title":"BASE Layers: Simplifying Training of Large, Sparse Models","created":"2021-03-30","cited_by_count":320},{"id":"2310.17157","title":"Deja Vu: Contextual Sparsity for Efficient LLMs at Inference Time","created":"2023-10-26","cited_by_count":252},{"id":"2301.12726","title":"Specializing Smaller Language Models towards Multi-Step Reasoning","created":"2023-01-30","cited_by_count":251},{"id":"2106.04426","title":"Hash Layers For Large Sparse Models","created":"2021-06-08","cited_by_count":237},{"id":"2211.15841","title":"MegaBlocks: Efficient Sparse Training with Mixture-of-Experts","created":"2022-11-29","cited_by_count":209},{"id":"2212.05055","title":"Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints","created":"2022-12-09","cited_by_count":209},{"id":"1910.04732","title":"Structured Pruning of Large Language Models","created":"2019-10-10","cited_by_count":203},{"id":"2206.02770","title":"Multimodal Contrastive Learning with LIMoE: the Language-Image Mixture of Experts","created":"2022-06-06","cited_by_count":203},{"id":"2012.15701","title":"BinaryBERT: Pushing the Limit of BERT Quantization","created":"2020-12-31","cited_by_count":178},{"id":"2204.00408","title":"Structured Pruning Learns Compact and Accurate Models","created":"2022-04-01","cited_by_count":170},{"id":"2112.05682","title":"Self-attention Does Not Need $O(n^2)$ Memory","created":"2021-12-10","cited_by_count":154},{"id":"2110.01786","title":"MoEfication: Transformer Feed-forward Layers are Mixtures of Experts","created":"2021-10-05","cited_by_count":152},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148},{"id":"2210.06313","title":"The Lazy Neuron Phenomenon: On Emergence of Activation Sparsity in Transformers","created":"2022-10-12","cited_by_count":122},{"id":"2106.03760","title":"DSelect-k: Differentiable Selection in the Mixture of Experts with Applications to Multi-Task Learning","created":"2021-06-07","cited_by_count":115},{"id":"2203.07259","title":"The Optimal BERT Surgeon: Scalable and Accurate Second-Order Pruning for Large Language Models","created":"2022-03-14","cited_by_count":115},{"id":"2110.03742","title":"Beyond Distillation: Task-level Mixture-of-Experts for Efficient Inference","created":"2021-09-24","cited_by_count":96},{"id":"2310.04564","title":"ReLU Strikes Back: Exploiting Activation Sparsity in Large Language Models","created":"2023-10-06","cited_by_count":94},{"id":"2111.12763","title":"Sparse is Enough in Scaling Transformers","created":"2021-11-24","cited_by_count":86},{"id":"2302.14017","title":"Full Stack Optimization of Transformer Inference: a Survey","created":"2023-02-27","cited_by_count":80},{"id":"2402.03804","title":"ReLU$^2$ Wins: Discovering Efficient Activation Functions for Sparse LLMs","created":"2024-02-06","cited_by_count":67},{"id":"2111.05754","title":"Prune Once for All: Sparse Pre-Trained Language Models","created":"2021-11-10","cited_by_count":66},{"id":"2210.05144","title":"Mixture of Attention Heads: Selecting Attention Heads Per Token","created":"2022-10-11","cited_by_count":60},{"id":"2009.14167","title":"Contrastive Distillation on Intermediate Representations for Language Model Compression","created":"2020-09-29","cited_by_count":57},{"id":"2402.13516","title":"ProSparse: Introducing and Enhancing Intrinsic Activation Sparsity within Large Language Models","created":"2024-02-21","cited_by_count":45},{"id":"2012.01266","title":"Meta-KD: A Meta Knowledge Distillation Framework for Language Model Compression across Domains","created":"2020-12-02","cited_by_count":33},{"id":"2311.05908","title":"FlashFFTConv: Efficient Convolutions for Long Sequences with Tensor Cores","created":"2023-11-10","cited_by_count":33},{"id":"2109.02008","title":"Cross-token Modeling with Conditional Computation","created":"2021-09-05","cited_by_count":17},{"id":"2310.04361","title":"Exploiting Activation Sparsity with Dense to Dynamic-k Mixture-of-Experts Conversion","created":"2023-10-06","cited_by_count":16},{"id":"1802.05371","title":"Input-Aware Auto-Tuning of Compute-Bound HPC Kernels","created":"2018-02-15","cited_by_count":5},{"id":"2209.02882","title":"Sgap: Towards Efficient Sparse Tensor Algebra Compilation for GPU","created":"2022-09-07","cited_by_count":2}],"next_cursor":null}