{"total":263,"rows":[{"id":"2401.06066","title":"DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models","created":"2024-01-11","cited_by_count":867,"area":"moe"},{"id":"2409.02060","title":"OLMoE: Open Mixture-of-Experts Language Models","created":"2024-09-03","cited_by_count":301,"area":"moe"},{"id":"2206.03382","title":"Tutel: Adaptive Mixture-of-Experts at Scale","created":"2022-06-07","cited_by_count":201,"area":"moe"},{"id":"2502.13189","title":"MoBA: Mixture of Block Attention for Long-Context LLMs","created":"2025-02-18","cited_by_count":198,"area":"moe"},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2404.15159","title":"MixLoRA: Enhancing Large Language Models Fine-Tuning with LoRA-based Mixture of Experts","created":"2024-04-22","cited_by_count":147,"area":"moe"},{"id":"2406.16554","title":"LLaMA-MoE: Building Mixture-of-Experts from LLaMA with Continual Pre-training","created":"2024-06-24","cited_by_count":123,"area":"moe"},{"id":"2210.17223","title":"Accelerating Distributed MoE Training and Inference with Lina","created":"2022-10-31","cited_by_count":109,"area":"moe"},{"id":"2310.01334","title":"Merge, Then Compress: Demystify Efficient SMoE with Hints from Its Routing Policy","created":"2023-10-02","cited_by_count":108,"area":"moe"},{"id":"2308.12066","title":"Pre-gated MoE: An Algorithm-System Co-Design for Fast and Scalable Mixture-of-Expert Inference","created":"2023-08-23","cited_by_count":104,"area":"moe"},{"id":"2410.10469","title":"Moirai-MoE: Empowering Time Series Foundation Models with Sparse Mixture of Experts","created":"2024-10-14","cited_by_count":95,"area":"moe"},{"id":"2206.00277","title":"Task-Specific Expert Pruning for Sparse Mixture-of-Experts","created":"2022-06-01","cited_by_count":72,"area":"moe"},{"id":"2309.14393","title":"LLMCarbon: Modeling the end-to-end Carbon Footprint of Large Language Models","created":"2023-09-25","cited_by_count":72,"area":"moe"},{"id":"2402.07033","title":"Fiddler: CPU-GPU Orchestration for Fast Inference of Mixture-of-Experts Models","created":"2024-02-10","cited_by_count":71,"area":"moe"},{"id":"2210.05144","title":"Mixture of Attention Heads: Selecting Attention Heads Per Token","created":"2022-10-11","cited_by_count":60,"area":"moe"},{"id":"2405.14297","title":"Dynamic Mixture of Experts: An Auto-Tuning Approach for Efficient Transformer Models","created":"2024-05-23","cited_by_count":56,"area":"moe"},{"id":"2410.07348","title":"MoE++: Accelerating Mixture-of-Experts Methods with Zero-Computation Experts","created":"2024-10-09","cited_by_count":56,"area":"moe"},{"id":"2501.11873","title":"Demons in the Detail: On Implementing Load Balancing Loss for Training Specialized Mixture-of-Expert Models","created":"2025-01-21","cited_by_count":54,"area":"moe"},{"id":"2304.03946","title":"FlexMoE: Scaling Large-scale Sparse Pre-trained Model Training via Dynamic Device Placement","created":"2023-04-08","cited_by_count":53,"area":"moe"},{"id":"2502.19811","title":"Comet: Fine-grained Computation-communication Overlapping for Mixture-of-Experts","created":"2025-02-27","cited_by_count":49,"area":"moe"},{"id":"2303.06318","title":"A Hybrid Tensor-Expert-Data Parallelism Approach to Optimize Mixture-of-Experts Training","created":"2023-03-11","cited_by_count":47,"area":"moe"},{"id":"2412.14219","title":"A Survey on Inference Optimization Techniques for Mixture of Experts Models","created":"2024-12-18","cited_by_count":47,"area":"moe"},{"id":"2410.11842","title":"MoH: Multi-Head Attention as Mixture-of-Head Attention","created":"2024-10-15","cited_by_count":44,"area":"moe"},{"id":"2310.18859","title":"SiDA-MoE: Sparsity-Inspired Data-Aware Serving for Efficient and Scalable Large Mixture-of-Experts Models","created":"2023-10-29","cited_by_count":38,"area":"moe"},{"id":"2406.05955","title":"Turbo Sparse: Achieving LLM SOTA Performance with Minimal Activated Parameters","created":"2024-06-10","cited_by_count":38,"area":"moe"},{"id":"2205.10034","title":"MoESys: A Distributed and Efficient Mixture-of-Experts Training and Inference System for Internet Services","created":"2022-05-20","cited_by_count":37,"area":"moe"},{"id":"2303.06182","title":"Towards MoE Deployment: Mitigating Inefficiencies in Mixture-of-Expert (MoE) Inference","created":"2023-03-10","cited_by_count":37,"area":"moe"},{"id":"2402.01771","title":"BlackMamba: Mixture of Experts for State-Space Models","created":"2024-02-01","cited_by_count":37,"area":"moe"},{"id":"2203.14685","title":"HetuMoE: An Efficient Trillion-scale Mixture-of-Expert Distributed Training System","created":"2022-03-28","cited_by_count":36,"area":"moe"},{"id":"2310.02410","title":"Mixture of Quantized Experts (MoQE): Complementary Effect of Low-bit Quantization and Robustness","created":"2023-10-03","cited_by_count":34,"area":"moe"},{"id":"2409.01141","title":"Duplex: A Device for Large Language Models with Mixture of Experts, Grouped Query Attention, and Continuous Batching","created":"2024-09-02","cited_by_count":34,"area":"moe"},{"id":"2404.05019","title":"Shortcut-connected Expert Parallelism for Accelerating Mixture-of-Experts","created":"2024-04-07","cited_by_count":32,"area":"moe"},{"id":"2504.02263","title":"MegaScale-Infer: Serving Mixture-of-Experts at Scale with Disaggregated Expert Parallelism","created":"2025-04-03","cited_by_count":32,"area":"moe"},{"id":"2203.01104","title":"Parameter-Efficient Mixture-of-Experts Architecture for Pre-trained Language Models","created":"2022-03-02","cited_by_count":31,"area":"moe"},{"id":"2505.11432","title":"MegaScale-MoE: Large-Scale Communication-Efficient Training of Mixture-of-Experts Models in Production","created":"2025-05-16","cited_by_count":31,"area":"moe"},{"id":"2310.09762","title":"Diversifying the Mixture-of-Experts Representation for Language Models with Orthogonal Optimizer","created":"2023-10-15","cited_by_count":29,"area":"moe"},{"id":"2401.08383","title":"Exploiting Inter-Layer Expert Affinity for Accelerating Mixture-of-Experts Model Inference","created":"2024-01-16","cited_by_count":28,"area":"moe"},{"id":"2403.18926","title":"XMoE: Sparse Models with Fine-grained and Adaptive Expert Selection","created":"2024-02-27","cited_by_count":28,"area":"moe"},{"id":"2211.10017","title":"Who Says Elephants Can't Run: Bringing Large Scale MoE Models into Cloud Scale Production","created":"2022-11-18","cited_by_count":27,"area":"moe"},{"id":"2408.11304","title":"FedMoE: Personalized Federated Learning via Heterogeneous Mixture of Experts","created":"2024-08-21","cited_by_count":27,"area":"moe"},{"id":"2402.12656","title":"HyperMoE: Towards Better Mixture of Experts via Transferring Among Experts","created":"2024-02-20","cited_by_count":26,"area":"moe"},{"id":"2411.15708","title":"LLaMA-MoE v2: Exploring Sparsity of LLaMA from Perspective of Mixture-of-Experts with Post-Training","created":"2024-11-24","cited_by_count":26,"area":"moe"},{"id":"2502.06643","title":"MoETuner: Optimized Mixture of Expert Serving with Balanced Expert Placement and Token Routing","created":"2025-02-10","cited_by_count":26,"area":"moe"},{"id":"2201.10890","title":"One Student Knows All Experts Know: From Sparse to Dense","created":"2022-01-26","cited_by_count":25,"area":"moe"},{"id":"2305.12129","title":"Lifting the Curse of Capacity Gap in Distilling Language Models","created":"2023-05-20","cited_by_count":25,"area":"moe"},{"id":"2410.05661","title":"Scaling Laws Across Model Architectures: A Comparative Analysis of Dense and MoE Models in Large Language Models","created":"2024-10-08","cited_by_count":25,"area":"moe"},{"id":"2406.02500","title":"Towards Efficient Mixture of Experts: A Holistic Study of Compression Techniques","created":"2024-06-04","cited_by_count":24,"area":"moe"},{"id":"2409.19291","title":"CLIP-MoE: Towards Building Mixture of Experts for CLIP with Diversified Multiplet Upcycling","created":"2024-09-28","cited_by_count":23,"area":"moe"},{"id":"2401.02731","title":"Parameter-Efficient Sparsity Crafting from Dense to Mixture-of-Experts for Instruction Tuning on General Tasks","created":"2024-01-05","cited_by_count":22,"area":"moe"},{"id":"2404.19429","title":"Lancet: Accelerating Mixture-of-Experts Training via Whole Graph Computation-Communication Overlapping","created":"2024-04-30","cited_by_count":22,"area":"moe"},{"id":"2503.05139","title":"Every FLOP Counts: Scaling a 300B Mixture-of-Experts LING LLM without Premium GPUs","created":"2025-03-07","cited_by_count":22,"area":"moe"},{"id":"2408.06793","title":"Layerwise Recurrent Router for Mixture-of-Experts","created":"2024-08-13","cited_by_count":21,"area":"moe"},{"id":"2410.19034","title":"Mixture of Parrots: Experts improve memorization more than reasoning","created":"2024-10-24","cited_by_count":21,"area":"moe"},{"id":"2502.06888","title":"Klotski: Efficient Mixture-of-Expert Inference via Expert-Aware Multi-Batch Pipeline","created":"2025-02-09","cited_by_count":21,"area":"moe"},{"id":"2504.05586","title":"Finding Fantastic Experts in MoEs: A Unified Study for Expert Dropping Strategies and Observations","created":"2025-04-08","cited_by_count":20,"area":"moe"},{"id":"2401.13920","title":"LocMoE: A Low-Overhead MoE for Large Language Model Training","created":"2024-01-25","cited_by_count":19,"area":"moe"},{"id":"2411.12364","title":"Ultra-Sparse Memory Network","created":"2024-11-19","cited_by_count":19,"area":"moe"},{"id":"2312.10868","title":"From Google Gemini to OpenAI Q* (Q-Star): A Survey of Reshaping the Generative Artificial Intelligence (AI) Research Landscape","created":"2023-12-18","cited_by_count":18,"area":"moe"},{"id":"2501.16352","title":"Mixture of Experts (MoE): A Big Data Perspective","created":"2025-01-18","cited_by_count":18,"area":"moe"},{"id":"2302.14703","title":"Improving Expert Specialization in Mixture of Experts","created":"2023-02-28","cited_by_count":17,"area":"moe"},{"id":"2501.10714","title":"FSMoE: A Flexible and Scalable Training System for Sparse Mixture-of-Experts Models","created":"2025-01-18","cited_by_count":17,"area":"moe"},{"id":"2504.14960","title":"MoE Parallel Folding: Heterogeneous Parallelism Mappings for Efficient Large-Scale MoE Model Training with Megatron Core","created":"2025-04-21","cited_by_count":17,"area":"moe"},{"id":"2310.07188","title":"Adaptive Gating in Mixture-of-Experts based Language Models","created":"2023-10-11","cited_by_count":16,"area":"moe"},{"id":"2504.06792","title":"Domain-Specific Pruning of Large Mixture-of-Experts Models with Few-shot Demonstrations","created":"2025-04-09","cited_by_count":16,"area":"moe"},{"id":"2302.09915","title":"TA-MoE: Topology-Aware Large Scale Mixture-of-Expert Training","created":"2023-02-20","cited_by_count":15,"area":"moe"},{"id":"2306.06446","title":"ShiftAddViT: Mixture of Multiplication Primitives Towards Efficient Vision Transformer","created":"2023-06-10","cited_by_count":15,"area":"moe"},{"id":"2409.06669","title":"DA-MoE: Towards Dynamic Expert Allocation for Mixture-of-Experts Models","created":"2024-09-10","cited_by_count":15,"area":"moe"},{"id":"2507.11181","title":"Mixture of Experts in Large Language Models","created":"2025-07-15","cited_by_count":15,"area":"moe"},{"id":"2204.10598","title":"Sparsely-gated Mixture-of-Expert Layers for CNN Interpretability","created":"2022-04-22","cited_by_count":14,"area":"moe"},{"id":"2412.20157","title":"UniRestorer: Universal Image Restoration via Adaptively Estimating Image Degradation at Proper Granularity","created":"2024-12-28","cited_by_count":14,"area":"moe"},{"id":"2506.04667","title":"FlashMoE: Fast Distributed MoE in a Single Kernel","created":"2025-06-05","cited_by_count":14,"area":"moe"},{"id":"2307.12169","title":"Rail-only: A Low-Cost High-Performance Network for Training LLMs with Trillion Parameters","created":"2023-07-22","cited_by_count":13,"area":"moe"},{"id":"2502.12224","title":"Fate: Fast Edge Inference of Mixture-of-Experts Models via Cross-Layer Gate","created":"2025-02-17","cited_by_count":13,"area":"moe"},{"id":"2308.06093","title":"Experts Weights Averaging: A New General Training Scheme for Vision Transformers","created":"2023-08-11","cited_by_count":12,"area":"moe"},{"id":"2402.12399","title":"Turn Waste into Worth: Rectifying Top-$k$ Router of MoE","created":"2024-02-17","cited_by_count":11,"area":"moe"},{"id":"2503.06823","title":"eMoE: Task-aware Memory Efficient Mixture-of-Experts-Based (MoE) Model Inference","created":"2025-03-10","cited_by_count":11,"area":"moe"},{"id":"2511.19822","title":"Mosaic Pruning: A Hierarchical Framework for Generalizable Pruning of Mixture-of-Experts Models","created":"2025-11-25","cited_by_count":11,"area":"moe"},{"id":"2210.07535","title":"AutoMoE: Heterogeneous Mixture-of-Experts with Adaptive Computation for Efficient Neural Machine Translation","created":"2022-10-14","cited_by_count":10,"area":"moe"},{"id":"2403.07965","title":"Conditional computation in neural networks: principles and research trends","created":"2024-03-12","cited_by_count":10,"area":"moe"},{"id":"2510.14436","title":"MergeMoE: Efficient Compression of MoE Models via Expert Output Merging","created":"2025-10-16","cited_by_count":10,"area":"moe"},{"id":"2510.17483","title":"ReXMoE: Reusing Experts with Minimal Overhead in Mixture-of-Experts","created":"2025-10-20","cited_by_count":10,"area":"moe"},{"id":"2406.04801","title":"MoE Jetpack: From Dense Checkpoints to Adaptive Mixture of Experts for Vision Tasks","created":"2024-06-07","cited_by_count":9,"area":"moe"},{"id":"2406.08155","title":"QuantMoE-Bench: Examining Post-Training Quantization for Mixture-of-Experts","created":"2024-06-12","cited_by_count":9,"area":"moe"},{"id":"2407.09816","title":"MaskMoE: Boosting Token-Level Learning via Routing Mask in Mixture-of-Experts","created":"2024-07-13","cited_by_count":9,"area":"moe"},{"id":"2505.03531","title":"Faster MoE LLM Inference for Extremely Large Models","created":"2025-05-06","cited_by_count":9,"area":"moe"},{"id":"2507.19017","title":"MindSpeed RL: Distributed Dataflow for Scalable and Efficient RL Training on Ascend NPU Cluster","created":"2025-07-25","cited_by_count":9,"area":"moe"},{"id":"2602.08621","title":"Sparse Models, Sparse Safety: Unsafe Routes in Mixture-of-Experts LLMs","created":"2026-02-09","cited_by_count":9,"area":"moe"},{"id":"2606.04101","title":"UltraEP: Unleash MoE Training and Inference on Rack-Scale Nodes with Near-Optimal Load Balancing","created":"2026-06-02","cited_by_count":9,"area":"moe"},{"id":"2212.07571","title":"Fixing MoE Over-Fitting on Low-Resource Languages in Multilingual Machine Translation","created":"2022-12-15","cited_by_count":8,"area":"moe"},{"id":"2406.12375","title":"GW-MoE: Resolving Uncertainty in MoE Router with Global Workspace Theory","created":"2024-06-18","cited_by_count":8,"area":"moe"},{"id":"2409.01483","title":"Revisiting SMoE Language Models by Evaluating Inefficiencies with Task Specific Expert Pruning","created":"2024-09-02","cited_by_count":8,"area":"moe"},{"id":"2504.02658","title":"MiLo: Efficient Quantized MoE Inference with Mixture of Low-Rank Compensators","created":"2025-04-03","cited_by_count":8,"area":"moe"},{"id":"2505.16056","title":"Not All Models Suit Expert Offloading: On Local Routing Consistency of Mixture-of-Expert Models","created":"2025-05-21","cited_by_count":8,"area":"moe"},{"id":"2509.23638","title":"LayerScope: Predictive Cross-Layer Scheduling for Efficient Multi-Batch MoE Inference on Legacy Servers","created":"2025-09-28","cited_by_count":8,"area":"moe"},{"id":"2406.12923","title":"Interpretable Cascading Mixture-of-Experts for Urban Traffic Congestion Prediction","created":"2024-06-14","cited_by_count":7,"area":"moe"},{"id":"2407.00599","title":"Parm: Efficient Training of Large Sparsely-Activated Models with Dedicated Schedules","created":"2024-06-30","cited_by_count":7,"area":"moe"},{"id":"2407.14417","title":"Mixture of Experts with Mixture of Precisions for Tuning Quality of Service","created":"2024-07-19","cited_by_count":7,"area":"moe"},{"id":"2409.18412","title":"SciDFM: A Large Language Model with Mixture-of-Experts for Science","created":"2024-09-27","cited_by_count":7,"area":"moe"},{"id":"2410.17043","title":"Optimizing Mixture-of-Experts Inference Time Combining Model Deployment and Communication Scheduling","created":"2024-10-22","cited_by_count":7,"area":"moe"},{"id":"2504.03871","title":"HeterMoE: Efficient Training of Mixture-of-Experts Models on Heterogeneous GPUs","created":"2025-04-04","cited_by_count":7,"area":"moe"}],"next_cursor":"100"}