{"total":253,"rows":[{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019,"area":null},{"id":"2303.18223","title":"A Survey of Large Language Models","created":"2023-03-31","cited_by_count":2576,"area":null},{"id":"2401.04088","title":"Mixtral of Experts","created":"2024-01-08","cited_by_count":1823,"area":"moe"},{"id":"2406.04093","title":"Scaling and evaluating sparse autoencoders","created":"2024-06-06","cited_by_count":553,"area":"mechinterp"},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535,"area":null},{"id":"2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","created":"2023-04-28","cited_by_count":523,"area":null},{"id":"2403.19887","title":"Jamba: A Hybrid Transformer-Mamba Language Model","created":"2024-03-28","cited_by_count":401,"area":"moe"},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399,"area":"moe"},{"id":"2308.07107","title":"Large Language Models for Information Retrieval: A Survey","created":"2023-08-14","cited_by_count":331,"area":null},{"id":"2409.02060","title":"OLMoE: Open Mixture-of-Experts Language Models","created":"2024-09-03","cited_by_count":301,"area":"moe"},{"id":"2308.00951","title":"From Sparse to Soft Mixtures of Experts","created":"2023-08-02","cited_by_count":248,"area":"moe"},{"id":"2205.10770","title":"Memorization Without Overfitting: Analyzing the Training Dynamics of Large Language Models","created":"2022-05-22","cited_by_count":220,"area":null},{"id":"2211.15841","title":"MegaBlocks: Efficient Sparse Training with Mixture-of-Experts","created":"2022-11-29","cited_by_count":209,"area":"moe"},{"id":"2212.05055","title":"Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints","created":"2022-12-09","cited_by_count":209,"area":"moe"},{"id":"2206.03382","title":"Tutel: Adaptive Mixture-of-Experts at Scale","created":"2022-06-07","cited_by_count":201,"area":"moe"},{"id":"2204.09179","title":"On the Representation Collapse of Sparse Mixture of Experts","created":"2022-04-20","cited_by_count":157,"area":"moe"},{"id":"2309.05444","title":"Pushing Mixture of Experts to the Limit: Extremely Parameter Efficient MoE for Instruction Tuning","created":"2023-09-11","cited_by_count":150,"area":"moe"},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2301.03728","title":"Scaling Laws for Generative Mixed-Modal Language Models","created":"2023-01-10","cited_by_count":134,"area":null},{"id":"2406.15479","title":"Twin-Merging: Dynamic Integration of Modular Expertise in Model Merging","created":"2024-06-17","cited_by_count":116,"area":null},{"id":"2210.14891","title":"Broken Neural Scaling Laws","created":"2022-10-26","cited_by_count":109,"area":null},{"id":"2210.14793","title":"M$^3$ViT: Mixture-of-Experts Vision Transformer for Efficient Multi-task Learning with Model-Accelerator Co-design","created":"2022-10-26","cited_by_count":98,"area":"moe"},{"id":"2411.04330","title":"Scaling Laws for Precision","created":"2024-11-07","cited_by_count":95,"area":null},{"id":"2403.15796","title":"Understanding Emergent Abilities of Language Models from the Loss Perspective","created":"2024-03-23","cited_by_count":94,"area":null},{"id":"2306.03745","title":"Soft Merging of Experts with Adaptive Routing","created":"2023-06-06","cited_by_count":90,"area":"moe"},{"id":"2404.14851","title":"From Matching to Generation: A Survey on Generative Information Retrieval","created":"2024-04-23","cited_by_count":89,"area":null},{"id":"2303.07226","title":"Scaling Vision-Language Models with Sparse Mixture of Experts","created":"2023-03-13","cited_by_count":88,"area":"mllm"},{"id":"2302.11529","title":"Modular Deep Learning","created":"2023-02-22","cited_by_count":83,"area":null},{"id":"2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","created":"2022-12-28","cited_by_count":81,"area":null},{"id":"2407.04153","title":"Mixture of A Million Experts","created":"2024-07-04","cited_by_count":72,"area":"moe"},{"id":"2305.14235","title":"Multilingual Large Language Models Are Not (Yet) Code-Switchers","created":"2023-05-23","cited_by_count":65,"area":null},{"id":"2305.14705","title":"Mixture-of-Experts Meets Instruction Tuning:A Winning Combination for Large Language Models","created":"2023-05-24","cited_by_count":65,"area":"moe"},{"id":"2304.02806","title":"Graph Mixture of Experts: Learning on Large-Scale Graphs with Explicit Diversity Modeling","created":"2023-04-06","cited_by_count":64,"area":"moe"},{"id":"2402.09748","title":"Model Compression and Efficient Inference for Large Language Models: A Survey","created":"2024-02-15","cited_by_count":64,"area":null},{"id":"2503.10657","title":"RouterEval: A Comprehensive Benchmark for Routing LLMs to Explore Model-level Scaling Up in LLMs","created":"2025-03-08","cited_by_count":61,"area":null},{"id":"2303.14177","title":"Scaling Expert Language Models with Unsupervised Domain Discovery","created":"2023-03-24","cited_by_count":60,"area":null},{"id":"2303.01610","title":"Sparse MoE as the New Dropout: Scaling Dense and Self-Slimmable Transformers","created":"2023-03-02","cited_by_count":59,"area":"moe"},{"id":"2405.16039","title":"MoEUT: Mixture-of-Experts Universal Transformers","created":"2024-05-25","cited_by_count":59,"area":"moe"},{"id":"2406.06563","title":"Skywork-MoE: A Deep Dive into Training Techniques for Mixture-of-Experts Language Models","created":"2024-06-03","cited_by_count":59,"area":"moe"},{"id":"2403.17749","title":"Multi-Task Dense Prediction via Mixture of Low-Rank Experts","created":"2024-03-26","cited_by_count":58,"area":"moe"},{"id":"2303.10845","title":"PanGu-{\\Sigma}: Towards Trillion Parameter Language Model with Sparse Heterogeneous Computing","created":"2023-03-20","cited_by_count":56,"area":null},{"id":"2405.14297","title":"Dynamic Mixture of Experts: An Auto-Tuning Approach for Efficient Transformer Models","created":"2024-05-23","cited_by_count":56,"area":"moe"},{"id":"2412.14711","title":"ReMoE: Fully Differentiable Mixture-of-Experts with ReLU Routing","created":"2024-12-19","cited_by_count":54,"area":"moe"},{"id":"2507.09404","title":"Scaling Laws for Optimal Data Mixtures","created":"2025-07-12","cited_by_count":52,"area":null},{"id":"2501.12370","title":"Parameters vs FLOPs: Scaling Laws for Optimal Sparsity for Mixture-of-Experts Language Models","created":"2025-01-21","cited_by_count":50,"area":"moe"},{"id":"2504.07951","title":"Scaling Laws for Native Multimodal Models","created":"2025-04-10","cited_by_count":50,"area":null},{"id":"2410.08527","title":"Scaling Laws for Predicting Downstream Performance in LLMs","created":"2024-10-11","cited_by_count":48,"area":null},{"id":"2309.04255","title":"LLMCad: Fast and Scalable On-device Large Language Model Inference","created":"2023-09-08","cited_by_count":45,"area":null},{"id":"2507.17702","title":"Towards Greater Leverage: Scaling Laws for Efficient Mixture-of-Experts Language Models","created":"2025-07-23","cited_by_count":45,"area":"moe"},{"id":"2307.03132","title":"T-MARS: Improving Visual Representations by Circumventing Text Feature Learning","created":"2023-07-06","cited_by_count":43,"area":null},{"id":"2309.08520","title":"Scaling Laws for Sparsely-Connected Foundation Models","created":"2023-09-15","cited_by_count":43,"area":null},{"id":"2404.00725","title":"The Larger the Better? Improved LLM Code-Generation via Budget Reallocation","created":"2024-03-31","cited_by_count":43,"area":null},{"id":"2502.17578","title":"How Do Large Language Monkeys Get Their Power (Laws)?","created":"2025-02-24","cited_by_count":43,"area":"safety"},{"id":"2310.16795","title":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","created":"2023-10-25","cited_by_count":42,"area":"moe"},{"id":"2406.01375","title":"D-CPT Law: Domain-specific Continual Pre-Training Scaling Law for Large Language Models","created":"2024-06-03","cited_by_count":41,"area":null},{"id":"2406.12907","title":"Reconciling Kaplan and Chinchilla Scaling Laws","created":"2024-06-12","cited_by_count":39,"area":null},{"id":"2210.00849","title":"Scaling Laws for a Multi-Agent Reinforcement Learning Model","created":"2022-09-29","cited_by_count":38,"area":"agents"},{"id":"2401.05605","title":"Scaling Laws for Forgetting When Fine-Tuning Large Language Models","created":"2024-01-11","cited_by_count":38,"area":null},{"id":"2402.01771","title":"BlackMamba: Mixture of Experts for State-Space Models","created":"2024-02-01","cited_by_count":37,"area":"moe"},{"id":"2312.07987","title":"SwitchHead: Accelerating Transformers with Mixture-of-Experts Attention","created":"2023-12-13","cited_by_count":34,"area":"moe"},{"id":"2309.08632","title":"Pretraining on the Test Set Is All You Need","created":"2023-09-13","cited_by_count":33,"area":null},{"id":"2406.15720","title":"Scaling Laws for Fact Memorization of Large Language Models","created":"2024-06-22","cited_by_count":33,"area":null},{"id":"2209.15466","title":"Sparsity-Constrained Optimal Transport","created":"2022-09-30","cited_by_count":32,"area":null},{"id":"2403.09176","title":"Switch Diffusion Transformer: Synergizing Denoising Tasks with Sparse Mixture-of-Experts","created":"2024-03-14","cited_by_count":31,"area":"diffusion"},{"id":"2404.15045","title":"Multi-Head Mixture-of-Experts","created":"2024-04-23","cited_by_count":31,"area":"moe"},{"id":"2403.18684","title":"Scaling Laws For Dense Retrieval","created":"2024-03-27","cited_by_count":29,"area":null},{"id":"2310.10837","title":"Approximating Two-Layer Feedforward Networks for Efficient Transformers","created":"2023-10-16","cited_by_count":27,"area":null},{"id":"2402.12656","title":"HyperMoE: Towards Better Mixture of Experts via Transferring Among Experts","created":"2024-02-20","cited_by_count":26,"area":"moe"},{"id":"2502.18969","title":"(Mis)Fitting: A Survey of Scaling Laws","created":"2025-02-26","cited_by_count":26,"area":null},{"id":"2410.05661","title":"Scaling Laws Across Model Architectures: A Comparative Analysis of Dense and MoE Models in Large Language Models","created":"2024-10-08","cited_by_count":25,"area":"moe"},{"id":"2202.07101","title":"A Survey on Dynamic Neural Networks for Natural Language Processing","created":"2022-02-15","cited_by_count":24,"area":null},{"id":"2403.06563","title":"Unraveling the Mystery of Scaling Laws: Part I","created":"2024-03-11","cited_by_count":24,"area":null},{"id":"2406.02500","title":"Towards Efficient Mixture of Experts: A Holistic Study of Compression Techniques","created":"2024-06-04","cited_by_count":24,"area":"moe"},{"id":"2503.18338","title":"SPMTrack: Spatio-Temporal Parameter-Efficient Fine-Tuning with Mixture of Experts for Scalable Visual Tracking","created":"2025-03-24","cited_by_count":24,"area":"moe"},{"id":"2401.15969","title":"Routers in Vision Mixture of Experts: An Empirical Study","created":"2024-01-29","cited_by_count":23,"area":"moe"},{"id":"2402.02526","title":"CompeteSMoE -- Effective Training of Sparse Mixture of Experts via Competition","created":"2024-02-04","cited_by_count":23,"area":"moe"},{"id":"2410.07265","title":"A Survey: Collaborative Hardware and Software Design in the Era of Large Language Models","created":"2024-10-08","cited_by_count":23,"area":null},{"id":"2503.05139","title":"Every FLOP Counts: Scaling a 300B Mixture-of-Experts LING LLM without Premium GPUs","created":"2025-03-07","cited_by_count":22,"area":"moe"},{"id":"2510.22037","title":"ATLAS: Adaptive Transfer Scaling Laws for Multilingual Pretraining, Finetuning, and Decoding the Curse of Multilinguality","created":"2025-10-24","cited_by_count":22,"area":null},{"id":"2410.19034","title":"Mixture of Parrots: Experts improve memorization more than reasoning","created":"2024-10-24","cited_by_count":21,"area":"moe"},{"id":"2512.14080","title":"SonicMoE: Accelerating MoE with IO and Tile-aware Optimizations","created":"2025-12-16","cited_by_count":21,"area":"moe"},{"id":"2402.13089","title":"Towards an empirical understanding of MoE design choices","created":"2024-02-20","cited_by_count":20,"area":"moe"},{"id":"2502.06857","title":"Gemstones: A Model Suite for Multi-Faceted Scaling Laws","created":"2025-02-07","cited_by_count":20,"area":null},{"id":"2306.13840","title":"Beyond Scale: The Diversity Coefficient as a Data Quality Metric for Variability in Natural Language Data","created":"2023-06-24","cited_by_count":19,"area":null},{"id":"2401.13920","title":"LocMoE: A Low-Overhead MoE for Large Language Model Training","created":"2024-01-25","cited_by_count":19,"area":"moe"},{"id":"2404.02852","title":"Toward Inference-optimal Mixture-of-Expert Large Language Models","created":"2024-04-03","cited_by_count":19,"area":"moe"},{"id":"2406.16690","title":"Scaling Laws for Linear Complexity Language Models","created":"2024-06-24","cited_by_count":19,"area":null},{"id":"2609.01343","title":"SMELT: Scaling Laws for Compute-Matched MoE Looped Transformers","created":"2026-09-01","cited_by_count":18,"area":"moe"},{"id":"2405.14260","title":"Graph Sparsification via Mixture of Graphs","created":"2024-05-23","cited_by_count":17,"area":"moe"},{"id":"2410.10114","title":"Mixture of Experts Made Personalized: Federated Prompt Learning for Vision-Language Models","created":"2024-10-14","cited_by_count":17,"area":"mllm"},{"id":"2310.04361","title":"Exploiting Activation Sparsity with Dense to Dynamic-k Mixture-of-Experts Conversion","created":"2023-10-06","cited_by_count":16,"area":"moe"},{"id":"2410.10456","title":"Ada-K Routing: Boosting the Efficiency of MoE-based LLMs","created":"2024-10-14","cited_by_count":16,"area":"moe"},{"id":"2508.12491","title":"Cost-Aware Contrastive Routing for LLMs","created":"2025-08-17","cited_by_count":16,"area":null},{"id":"2403.01197","title":"DMoERM: Recipes of Mixture-of-Experts for Effective Reward Modeling","created":"2024-03-02","cited_by_count":15,"area":"alignment"},{"id":"2204.10598","title":"Sparsely-gated Mixture-of-Expert Layers for CNN Interpretability","created":"2022-04-22","cited_by_count":14,"area":"moe"},{"id":"2210.13447","title":"Precision Machine Learning","created":"2022-10-24","cited_by_count":14,"area":null},{"id":"2211.13491","title":"Spatial Mixture-of-Experts","created":"2022-11-24","cited_by_count":14,"area":"moe"},{"id":"2412.20157","title":"UniRestorer: Universal Image Restoration via Adaptively Estimating Image Degradation at Proper Granularity","created":"2024-12-28","cited_by_count":14,"area":"moe"},{"id":"2502.19227","title":"Enhancing the Scalability and Applicability of Kohn-Sham Hamiltonians for Molecular Systems","created":"2025-02-26","cited_by_count":14,"area":null},{"id":"2504.02398","title":"Scaling Analysis of Interleaved Speech-Text Language Models","created":"2025-04-03","cited_by_count":14,"area":null}],"next_cursor":"100"}