{"total":246,"rows":[{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073,"area":"moe"},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631,"area":"moe"},{"id":"2106.05974","title":"Scaling Vision with Sparse Mixture of Experts","created":"2021-06-10","cited_by_count":747,"area":"moe"},{"id":"2202.09368","title":"Mixture-of-Experts with Expert Choice Routing","created":"2022-02-18","cited_by_count":623,"area":"moe"},{"id":"2201.05596","title":"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale","created":"2022-01-14","cited_by_count":404,"area":"moe"},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399,"area":"moe"},{"id":"2409.02060","title":"OLMoE: Open Mixture-of-Experts Language Models","created":"2024-09-03","cited_by_count":301,"area":"moe"},{"id":"2409.16040","title":"Time-MoE: Billion-Scale Time Series Foundation Models with Mixture of Experts","created":"2024-09-24","cited_by_count":287,"area":"moe"},{"id":"2212.05055","title":"Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints","created":"2022-12-09","cited_by_count":209,"area":"moe"},{"id":"2206.02770","title":"Multimodal Contrastive Learning with LIMoE: the Language-Image Mixture of Experts","created":"2022-06-06","cited_by_count":203,"area":"moe"},{"id":"2402.01739","title":"OpenMoE: An Early Effort on Open Mixture-of-Experts Language Models","created":"2024-01-29","cited_by_count":187,"area":"moe"},{"id":"2112.10684","title":"Efficient Large Scale Language Modeling with Mixtures of Experts","created":"2021-12-20","cited_by_count":175,"area":"moe"},{"id":"2402.07871","title":"Scaling Laws for Fine-Grained Mixture of Experts","created":"2024-02-12","cited_by_count":173,"area":"moe"},{"id":"2204.09179","title":"On the Representation Collapse of Sparse Mixture of Experts","created":"2022-04-20","cited_by_count":157,"area":"moe"},{"id":"2110.01786","title":"MoEfication: Transformer Feed-forward Layers are Mixtures of Experts","created":"2021-10-05","cited_by_count":152,"area":"moe"},{"id":"2309.05444","title":"Pushing Mixture of Experts to the Limit: Extremely Parameter Efficient MoE for Instruction Tuning","created":"2023-09-11","cited_by_count":150,"area":"moe"},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2402.14800","title":"Not All Experts are Equal: Efficient Expert Pruning and Skipping for Mixture-of-Experts Large Language Models","created":"2024-02-22","cited_by_count":146,"area":"moe"},{"id":"2406.16554","title":"LLaMA-MoE: Building Mixture-of-Experts from LLaMA with Continual Pre-training","created":"2024-06-24","cited_by_count":123,"area":"moe"},{"id":"2110.04260","title":"Taming Sparsely Activated Transformer with Stochastic Experts","created":"2021-10-08","cited_by_count":113,"area":"moe"},{"id":"2103.13262","title":"FastMoE: A Fast Mixture-of-Expert Training System","created":"2021-03-24","cited_by_count":112,"area":"moe"},{"id":"2210.17223","title":"Accelerating Distributed MoE Training and Inference with Lina","created":"2022-10-31","cited_by_count":109,"area":"moe"},{"id":"2310.01334","title":"Merge, Then Compress: Demystify Efficient SMoE with Hints from Its Routing Policy","created":"2023-10-02","cited_by_count":108,"area":"moe"},{"id":"2308.12066","title":"Pre-gated MoE: An Algorithm-System Co-Design for Fast and Scalable Mixture-of-Expert Inference","created":"2023-08-23","cited_by_count":104,"area":"moe"},{"id":"2204.08396","title":"StableMoE: Stable Routing Strategy for Mixture of Experts","created":"2022-04-18","cited_by_count":100,"area":"moe"},{"id":"2110.03742","title":"Beyond Distillation: Task-level Mixture-of-Experts for Efficient Inference","created":"2021-09-24","cited_by_count":96,"area":"moe"},{"id":"2305.13230","title":"To Repeat or Not To Repeat: Insights from Scaling LLM under Token-Crisis","created":"2023-05-22","cited_by_count":92,"area":"moe"},{"id":"2312.17238","title":"Fast Inference of Mixture-of-Experts Language Models with Offloading","created":"2023-12-28","cited_by_count":91,"area":"moe"},{"id":"2306.03745","title":"Soft Merging of Experts with Adaptive Routing","created":"2023-06-06","cited_by_count":90,"area":"moe"},{"id":"2109.10465","title":"Scalable and Efficient MoE Training for Multitask Multilingual Models","created":"2021-09-22","cited_by_count":76,"area":"moe"},{"id":"2309.14393","title":"LLMCarbon: Modeling the end-to-end Carbon Footprint of Large Language Models","created":"2023-09-25","cited_by_count":72,"area":"moe"},{"id":"2407.04153","title":"Mixture of A Million Experts","created":"2024-07-04","cited_by_count":72,"area":"moe"},{"id":"2402.03226","title":"FuseMoE: Mixture-of-Experts Transformers for Fleximodal Fusion","created":"2024-02-05","cited_by_count":68,"area":"moe"},{"id":"2305.12281","title":"Lifelong Language Pretraining with Distribution-Specialized Experts","created":"2023-05-20","cited_by_count":66,"area":"moe"},{"id":"2305.14705","title":"Mixture-of-Experts Meets Instruction Tuning:A Winning Combination for Large Language Models","created":"2023-05-24","cited_by_count":65,"area":"moe"},{"id":"2410.12013","title":"MoE-Pruner: Pruning Mixture-of-Experts Large Language Model using the Hints from Its Router","created":"2024-10-15","cited_by_count":62,"area":"moe"},{"id":"2203.06569","title":"SummaReranker: A Multi-Task Mixture-of-Experts Re-ranking Framework for Abstractive Summarization","created":"2022-03-13","cited_by_count":61,"area":"moe"},{"id":"2210.05144","title":"Mixture of Attention Heads: Selecting Attention Heads Per Token","created":"2022-10-11","cited_by_count":60,"area":"moe"},{"id":"2303.01610","title":"Sparse MoE as the New Dropout: Scaling Dense and Self-Slimmable Transformers","created":"2023-03-02","cited_by_count":59,"area":"moe"},{"id":"2405.16039","title":"MoEUT: Mixture-of-Experts Universal Transformers","created":"2024-05-25","cited_by_count":59,"area":"moe"},{"id":"2107.11817","title":"Go Wider Instead of Deeper","created":"2021-07-25","cited_by_count":57,"area":"moe"},{"id":"2404.07413","title":"JetMoE: Reaching Llama2 Performance with 0.1M Dollars","created":"2024-04-11","cited_by_count":57,"area":"moe"},{"id":"2410.07348","title":"MoE++: Accelerating Mixture-of-Experts Methods with Zero-Computation Experts","created":"2024-10-09","cited_by_count":56,"area":"moe"},{"id":"2304.03946","title":"FlexMoE: Scaling Large-scale Sparse Pre-trained Model Training via Dynamic Device Placement","created":"2023-04-08","cited_by_count":53,"area":"moe"},{"id":"2303.06318","title":"A Hybrid Tensor-Expert-Data Parallelism Approach to Optimize Mixture-of-Experts Training","created":"2023-03-11","cited_by_count":47,"area":"moe"},{"id":"2412.14219","title":"A Survey on Inference Optimization Techniques for Mixture of Experts Models","created":"2024-12-18","cited_by_count":47,"area":"moe"},{"id":"2410.11842","title":"MoH: Multi-Head Attention as Mixture-of-Head Attention","created":"2024-10-15","cited_by_count":44,"area":"moe"},{"id":"2310.16795","title":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","created":"2023-10-25","cited_by_count":42,"area":"moe"},{"id":"2410.06270","title":"Mixture Compressor for Mixture-of-Experts LLMs Gains More","created":"2024-10-08","cited_by_count":41,"area":"moe"},{"id":"2204.09636","title":"Residual Mixture of Experts","created":"2022-04-20","cited_by_count":39,"area":"moe"},{"id":"2208.07638","title":"Mask and Reason: Pre-Training Knowledge Graph Transformers for Complex Logical Queries","created":"2022-08-16","cited_by_count":39,"area":"moe"},{"id":"2310.18859","title":"SiDA-MoE: Sparsity-Inspired Data-Aware Serving for Efficient and Scalable Large Mixture-of-Experts Models","created":"2023-10-29","cited_by_count":38,"area":"moe"},{"id":"2204.07675","title":"MoEBERT: from BERT to Mixture-of-Experts via Importance-Guided Adaptation","created":"2022-04-15","cited_by_count":37,"area":"moe"},{"id":"2310.09832","title":"Merging Experts into One: Improving Computational Efficiency of Mixture of Experts","created":"2023-10-15","cited_by_count":37,"area":"moe"},{"id":"2405.03133","title":"Lory: Fully Differentiable Mixture-of-Experts for Autoregressive Language Model Pre-training","created":"2024-05-06","cited_by_count":37,"area":"moe"},{"id":"2407.09590","title":"Diversifying the Expert Knowledge for Task-Agnostic Pruning in Sparse Mixture-of-Experts","created":"2024-07-12","cited_by_count":37,"area":"moe"},{"id":"2203.14685","title":"HetuMoE: An Efficient Trillion-scale Mixture-of-Expert Distributed Training System","created":"2022-03-28","cited_by_count":36,"area":"moe"},{"id":"2410.08589","title":"Retraining-Free Merging of Sparse MoE via Hierarchical Clustering","created":"2024-10-11","cited_by_count":35,"area":"moe"},{"id":"2312.07987","title":"SwitchHead: Accelerating Transformers with Mixture-of-Experts Attention","created":"2023-12-13","cited_by_count":34,"area":"moe"},{"id":"2505.03804","title":"MoEQuant: Enhancing Quantization for Mixture-of-Experts Large Language Models via Expert-Balanced Sampling and Affinity Guidance","created":"2025-05-02","cited_by_count":34,"area":"moe"},{"id":"2502.17298","title":"Delta Decompression for MoE-based LLMs Compression","created":"2025-02-24","cited_by_count":33,"area":"moe"},{"id":"2203.01104","title":"Parameter-Efficient Mixture-of-Experts Architecture for Pre-trained Language Models","created":"2022-03-02","cited_by_count":31,"area":"moe"},{"id":"2404.15045","title":"Multi-Head Mixture-of-Experts","created":"2024-04-23","cited_by_count":31,"area":"moe"},{"id":"2112.14397","title":"EvoMoE: An Evolutional Mixture-of-Experts Training Framework via Dense-To-Sparse Gate","created":"2021-12-29","cited_by_count":30,"area":"moe"},{"id":"2308.10110","title":"Robust Mixture-of-Expert Training for Convolutional Neural Networks","created":"2023-08-19","cited_by_count":30,"area":"moe"},{"id":"2409.06211","title":"STUN: Structured-Then-Unstructured Pruning for Scalable MoE Pruning","created":"2024-09-10","cited_by_count":30,"area":"moe"},{"id":"2408.10681","title":"HMoE: Heterogeneous Mixture of Experts for Language Modeling","created":"2024-08-20","cited_by_count":29,"area":"moe"},{"id":"2403.18926","title":"XMoE: Sparse Models with Fine-grained and Adaptive Expert Selection","created":"2024-02-27","cited_by_count":28,"area":"moe"},{"id":"2402.12656","title":"HyperMoE: Towards Better Mixture of Experts via Transferring Among Experts","created":"2024-02-20","cited_by_count":26,"area":"moe"},{"id":"2502.06643","title":"MoETuner: Optimized Mixture of Expert Serving with Balanced Expert Placement and Token Routing","created":"2025-02-10","cited_by_count":26,"area":"moe"},{"id":"2509.16105","title":"DiEP: Adaptive Mixture-of-Experts Compression through Differentiable Expert Pruning","created":"2025-09-19","cited_by_count":26,"area":"moe"},{"id":"2305.12129","title":"Lifting the Curse of Capacity Gap in Distilling Language Models","created":"2023-05-20","cited_by_count":25,"area":"moe"},{"id":"2403.11808","title":"Dynamic Tuning Towards Parameter and Inference Efficiency for ViT Adaptation","created":"2024-03-18","cited_by_count":25,"area":"moe"},{"id":"2407.19985","title":"Mixture of Nested Experts: Adaptive Processing of Visual Tokens","created":"2024-07-29","cited_by_count":25,"area":"moe"},{"id":"2509.10377","title":"Dropping Experts, Recombining Neurons: Retraining-Free Pruning for Sparse Mixture-of-Experts LLMs","created":"2025-09-12","cited_by_count":25,"area":"moe"},{"id":"2210.16433","title":"Knowledge-in-Context: Towards Knowledgeable Semi-Parametric Language Models","created":"2022-10-28","cited_by_count":24,"area":"moe"},{"id":"2406.02500","title":"Towards Efficient Mixture of Experts: A Holistic Study of Compression Techniques","created":"2024-06-04","cited_by_count":24,"area":"moe"},{"id":"2502.19261","title":"Drop-Upcycling: Training Sparse Mixture of Experts with Partial Re-initialization","created":"2025-02-26","cited_by_count":24,"area":"moe"},{"id":"2105.06597","title":"RetGen: A Joint framework for Retrieval and Grounded Text Generation Modeling","created":"2021-05-14","cited_by_count":22,"area":"moe"},{"id":"2105.15082","title":"M6-T: Exploring Sparse Expert Models and Beyond","created":"2021-05-31","cited_by_count":22,"area":"moe"},{"id":"2205.14336","title":"Gating Dropout: Communication-efficient Regularization for Sparsely Activated Transformers","created":"2022-05-28","cited_by_count":22,"area":"moe"},{"id":"2308.09723","title":"FineQuant: Unlocking Efficiency with Fine-Grained Weight-Only Quantization for LLMs","created":"2023-08-16","cited_by_count":22,"area":"moe"},{"id":"2504.15299","title":"D$^{2}$MoE: Dual Routing and Dynamic Scheduling for Efficient On-Device MoE-based LLM Serving","created":"2025-04-17","cited_by_count":20,"area":"moe"},{"id":"2305.06324","title":"Alternating Gradient Descent and Mixture-of-Experts for Integrated Multimodal Perception","created":"2023-05-10","cited_by_count":19,"area":"moe"},{"id":"2410.19123","title":"Read-ME: Refactorizing LLMs as Router-Decoupled Mixture of Experts with System Co-Design","created":"2024-10-24","cited_by_count":19,"area":"moe"},{"id":"2411.12364","title":"Ultra-Sparse Memory Network","created":"2024-11-19","cited_by_count":19,"area":"moe"},{"id":"2508.01625","title":"EAC-MoE: Expert-Selection Aware Compressor for Mixture-of-Experts Large Language Models","created":"2025-08-03","cited_by_count":18,"area":"moe"},{"id":"2310.00811","title":"Sparse Backpropagation for MoE Training","created":"2023-10-01","cited_by_count":17,"area":"moe"},{"id":"2410.10626","title":"Efficiently Democratizing Medical LLMs for 50 Languages via a Mixture of Language Family Experts","created":"2024-10-14","cited_by_count":17,"area":"moe"},{"id":"2507.23279","title":"Unveiling Super Experts in Mixture-of-Experts Large Language Models","created":"2025-07-31","cited_by_count":17,"area":"moe"},{"id":"2310.04361","title":"Exploiting Activation Sparsity with Dense to Dynamic-k Mixture-of-Experts Conversion","created":"2023-10-06","cited_by_count":16,"area":"moe"},{"id":"2405.14131","title":"Statistical Advantages of Perturbing Cosine Router in Mixture of Experts","created":"2024-05-23","cited_by_count":16,"area":"moe"},{"id":"2410.10456","title":"Ada-K Routing: Boosting the Efficiency of MoE-based LLMs","created":"2024-10-14","cited_by_count":16,"area":"moe"},{"id":"2409.06669","title":"DA-MoE: Towards Dynamic Expert Allocation for Mixture-of-Experts Models","created":"2024-09-10","cited_by_count":15,"area":"moe"},{"id":"2207.09094","title":"MoEC: Mixture of Expert Clusters","created":"2022-07-19","cited_by_count":14,"area":"moe"},{"id":"2505.00315","title":"Mixture of Sparse Attention: Content-Based Learnable Sparse Attention via Expert-Choice Routing","created":"2025-05-01","cited_by_count":14,"area":"moe"},{"id":"2503.11880","title":"FedALT: Federated Fine-Tuning through Adaptive Local Training with Rest-of-World LoRA","created":"2025-03-14","cited_by_count":13,"area":"moe"},{"id":"2507.00390","title":"MoNE: Replacing Redundant Experts with Lightweight Novices for Structured Pruning of MoE","created":"2025-07-01","cited_by_count":13,"area":"moe"},{"id":"2601.20205","title":"Hyperparameter Transfer with Mixture-of-Expert Layers","created":"2026-01-28","cited_by_count":13,"area":"moe"},{"id":"2205.12399","title":"Sparse Mixers: Combining MoE and Mixing to build a more efficient BERT","created":"2022-05-24","cited_by_count":12,"area":"moe"}],"next_cursor":"100"}