{"list":"other","total":79,"rows":[{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2110.14168","title":"Training Verifiers to Solve Math Word Problems","created":"2021-10-27","cited_by_count":10109},{"id":"2107.03374","title":"Evaluating Large Language Models Trained on Code","created":"2021-07-07","cited_by_count":8320},{"id":"2009.03300","title":"Measuring Massive Multitask Language Understanding","created":"2020-09-07","cited_by_count":7963},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2103.03874","title":"Measuring Mathematical Problem Solving With the MATH Dataset","created":"2021-03-05","cited_by_count":5371},{"id":"2312.00752","title":"Mamba: Linear-Time Sequence Modeling with Selective State Spaces","created":"2023-12-01","cited_by_count":4718},{"id":"1606.08415","title":"Gaussian Error Linear Units (GELUs)","created":"2016-06-27","cited_by_count":3502},{"id":"2310.06825","title":"Mistral 7B","created":"2023-10-10","cited_by_count":3164},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2304.10592","title":"MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models","created":"2023-04-20","cited_by_count":2711},{"id":"2206.07682","title":"Emergent Abilities of Large Language Models","created":"2022-06-15","cited_by_count":2512},{"id":"1308.3432","title":"Estimating or Propagating Gradients Through Stochastic Neurons for Conditional Computation","created":"2013-08-15","cited_by_count":2079},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"1904.01038","title":"fairseq: A Fast, Extensible Toolkit for Sequence Modeling","created":"2019-04-01","cited_by_count":1488},{"id":"2406.00515","title":"A Survey on Large Language Models for Code Generation","created":"2024-06-01","cited_by_count":658},{"id":"2403.14608","title":"Parameter-Efficient Fine-Tuning for Large Models: A Comprehensive Survey","created":"2024-03-21","cited_by_count":656},{"id":"2401.02954","title":"DeepSeek LLM: Scaling Open-Source Language Models with Longtermism","created":"2024-01-05","cited_by_count":642},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"2401.15947","title":"MoE-LLaVA: Mixture of Experts for Large Vision-Language Models","created":"2024-01-29","cited_by_count":300},{"id":"1312.4314","title":"Learning Factored Representations in a Deep Mixture of Experts","created":"2013-12-16","cited_by_count":278},{"id":"2404.02258","title":"Mixture-of-Depths: Dynamically allocating compute in transformer-based language models","created":"2024-04-02","cited_by_count":258},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245},{"id":"2403.09611","title":"MM1: Methods, Analysis & Insights from Multimodal LLM Pre-training","created":"2024-03-14","cited_by_count":232},{"id":"2203.06904","title":"Delta Tuning: A Comprehensive Study of Parameter Efficient Methods for Pre-trained Language Models","created":"2022-03-14","cited_by_count":180},{"id":"2303.15647","title":"Scaling Down to Scale Up: A Guide to Parameter-Efficient Fine-Tuning","created":"2023-03-28","cited_by_count":173},{"id":"1511.06297","title":"Conditional Computation in Neural Networks for faster models","created":"2015-11-19","cited_by_count":167},{"id":"2309.05444","title":"Pushing Mixture of Experts to the Limit: Extremely Parameter Efficient MoE for Instruction Tuning","created":"2023-09-11","cited_by_count":150},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148},{"id":"2404.15159","title":"MixLoRA: Enhancing Large Language Models Fine-Tuning with LoRA-based Mixture of Experts","created":"2024-04-22","cited_by_count":147},{"id":"1711.01239","title":"Routing Networks: Adaptive Selection of Non-linear Functions for Multi-Task Learning","created":"2017-11-03","cited_by_count":126},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126},{"id":"2210.06313","title":"The Lazy Neuron Phenomenon: On Emergence of Activation Sparsity in Transformers","created":"2022-10-12","cited_by_count":122},{"id":"2103.13262","title":"FastMoE: A Fast Mixture-of-Expert Training System","created":"2021-03-24","cited_by_count":112},{"id":"2310.19341","title":"Skywork: A More Open Bilingual Foundation Model","created":"2023-10-30","cited_by_count":112},{"id":"2310.18339","title":"When MOE Meets LLMs: Parameter Efficient Fine-tuning for Multi-task Medical Applications","created":"2023-10-21","cited_by_count":111},{"id":"2405.11273","title":"Uni-MoE: Scaling Unified Multimodal LLMs with Mixture of Experts","created":"2024-05-18","cited_by_count":108},{"id":"2312.12379","title":"Mixture of Cluster-conditional LoRA Experts for Vision-language Instruction Tuning","created":"2023-12-19","cited_by_count":106},{"id":"2308.12066","title":"Pre-gated MoE: An Algorithm-System Co-Design for Fast and Scalable Mixture-of-Expert Inference","created":"2023-08-23","cited_by_count":104},{"id":"2402.08562","title":"Higher Layers Need More LoRA Experts","created":"2024-02-13","cited_by_count":102},{"id":"2306.03745","title":"Soft Merging of Experts with Adaptive Routing","created":"2023-06-06","cited_by_count":90},{"id":"2303.07226","title":"Scaling Vision-Language Models with Sparse Mixture of Experts","created":"2023-03-13","cited_by_count":88},{"id":"2401.16160","title":"LLaVA-MoLE: Sparse Mixture of LoRA Experts for Mitigating Data Conflicts in Instruction Finetuning MLLMs","created":"2024-01-29","cited_by_count":84},{"id":"2312.09979","title":"LoRAMoE: Alleviate World Knowledge Forgetting in Large Language Models via MoE-Style Plugin","created":"2023-12-15","cited_by_count":82},{"id":"2206.00277","title":"Task-Specific Expert Pruning for Sparse Mixture-of-Experts","created":"2022-06-01","cited_by_count":72},{"id":"2308.14352","title":"EdgeMoE: Empowering Sparse Large Language Models on Mobile Devices","created":"2023-08-28","cited_by_count":69},{"id":"2303.10845","title":"PanGu-{\\Sigma}: Towards Trillion Parameter Language Model with Sparse Heterogeneous Computing","created":"2023-03-20","cited_by_count":56},{"id":"2402.12851","title":"MoELoRA: Contrastive Learning Guided Mixture of Experts on Parameter-Efficient Fine-Tuning for Large Language Models","created":"2024-02-20","cited_by_count":56},{"id":"2405.14297","title":"Dynamic Mixture of Experts: An Auto-Tuning Approach for Efficient Transformer Models","created":"2024-05-23","cited_by_count":56},{"id":"2208.02813","title":"Towards Understanding Mixture of Experts in Deep Learning","created":"2022-08-04","cited_by_count":52},{"id":"1904.12774","title":"Routing Networks and the Challenges of Modular and Compositional Computation","created":"2019-04-29","cited_by_count":50},{"id":"1312.4461","title":"Low-Rank Approximations for Conditional Feedforward Computation in Deep Neural Networks","created":"2013-12-16","cited_by_count":44},{"id":"2406.13233","title":"AdaMoE: Token-Adaptive Routing with Null Experts for Mixture-of-Experts Language Models","created":"2024-06-19","cited_by_count":44},{"id":"2204.09636","title":"Residual Mixture of Experts","created":"2022-04-20","cited_by_count":39},{"id":"2205.10034","title":"MoESys: A Distributed and Efficient Mixture-of-Experts Training and Inference System for Internet Services","created":"2022-05-20","cited_by_count":37},{"id":"2203.14685","title":"HetuMoE: An Efficient Trillion-scale Mixture-of-Expert Distributed Training System","created":"2022-03-28","cited_by_count":36},{"id":"2311.09179","title":"SiRA: Sparse Mixture of Low Rank Adaptation","created":"2023-11-15","cited_by_count":32},{"id":"2404.05019","title":"Shortcut-connected Expert Parallelism for Accelerating Mixture-of-Experts","created":"2024-04-07","cited_by_count":32},{"id":"2112.14397","title":"EvoMoE: An Evolutional Mixture-of-Experts Training Framework via Dense-To-Sparse Gate","created":"2021-12-29","cited_by_count":30},{"id":"2401.08383","title":"Exploiting Inter-Layer Expert Affinity for Accelerating Mixture-of-Experts Model Inference","created":"2024-01-16","cited_by_count":28},{"id":"2312.00968","title":"Omni-SMoLA: Boosting Generalist Multimodal Models with Soft Mixture of Low-rank Experts","created":"2023-12-01","cited_by_count":26},{"id":"2201.10890","title":"One Student Knows All Experts Know: From Sparse to Dense","created":"2022-01-26","cited_by_count":25},{"id":"2105.15082","title":"M6-T: Exploring Sparse Expert Models and Beyond","created":"2021-05-31","cited_by_count":22},{"id":"2404.19429","title":"Lancet: Accelerating Mixture-of-Experts Training via Whole Graph Computation-Communication Overlapping","created":"2024-04-30","cited_by_count":22},{"id":"2405.14507","title":"Unchosen Experts Can Contribute Too: Unleashing MoE Models' Power by Self-Contrast","created":"2024-05-23","cited_by_count":19},{"id":"2404.01954","title":"HyperCLOVA X Technical Report","created":"2024-04-02","cited_by_count":18},{"id":"2405.17976","title":"Yuan 2.0-M32: Mixture of Experts with Attention Router","created":"2024-05-28","cited_by_count":18},{"id":"2405.13053","title":"MeteoRA: Multiple-tasks Embedded LoRA for Large Language Models","created":"2024-05-19","cited_by_count":16},{"id":"2305.06942","title":"Optimizing Distributed ML Communication with Fused Computation-Collective Operations","created":"2023-05-11","cited_by_count":15},{"id":"2403.08245","title":"Scattered Mixture-of-Experts Implementation","created":"2024-03-13","cited_by_count":15},{"id":"2405.14488","title":"MoGU: A Framework for Enhancing Safety of Open-Sourced LLMs While Preserving Their Usability","created":"2024-05-23","cited_by_count":15},{"id":"2308.06093","title":"Experts Weights Averaging: A New General Training Scheme for Vision Transformers","created":"2023-08-11","cited_by_count":12},{"id":"2404.08985","title":"T-REX: Mixture-of-Rank-One-Experts with Semantic-aware Intuition for Multi-task Large Language Model Finetuning","created":"2024-04-13","cited_by_count":8},{"id":"2310.15961","title":"Mixture of Tokens: Continuous MoE through Cross-Example Aggregation","created":"2023-10-24","cited_by_count":5},{"id":"2311.14747","title":"HOPE: A Memory-Based and Composition-Aware Framework for Zero-Shot Learning with Hopfield Network and Soft Mixture of Experts","created":"2023-11-23","cited_by_count":3}],"next_cursor":null}