{"total":175,"rows":[{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512,"area":null},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019,"area":null},{"id":"2206.07682","title":"Emergent Abilities of Large Language Models","created":"2022-06-15","cited_by_count":2512,"area":null},{"id":"2202.12837","title":"Rethinking the Role of Demonstrations: What Makes In-Context Learning Work?","created":"2022-02-25","cited_by_count":1434,"area":"lr"},{"id":"2211.01324","title":"eDiff-I: Text-to-Image Diffusion Models with an Ensemble of Expert Denoisers","created":"2022-11-02","cited_by_count":931,"area":"diffusion"},{"id":"2208.07339","title":"LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale","created":"2022-08-15","cited_by_count":906,"area":null},{"id":"2401.06066","title":"DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models","created":"2024-01-11","cited_by_count":867,"area":"moe"},{"id":"2210.02414","title":"GLM-130B: An Open Bilingual Pre-trained Model","created":"2022-10-05","cited_by_count":843,"area":null},{"id":"2306.01116","title":"The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only","created":"2023-06-01","cited_by_count":763,"area":null},{"id":"2204.06745","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","created":"2022-04-14","cited_by_count":713,"area":null},{"id":"2311.16867","title":"The Falcon Series of Open Language Models","created":"2023-11-28","cited_by_count":535,"area":null},{"id":"2204.05999","title":"InCoder: A Generative Model for Code Infilling and Synthesis","created":"2022-04-12","cited_by_count":498,"area":null},{"id":"2207.04429","title":"LM-Nav: Robotic Navigation with Large Pre-Trained Models of Language, Vision, and Action","created":"2022-07-10","cited_by_count":439,"area":null},{"id":"2201.05596","title":"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale","created":"2022-01-14","cited_by_count":404,"area":"moe"},{"id":"2202.08906","title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","created":"2022-02-17","cited_by_count":399,"area":"moe"},{"id":"2409.02060","title":"OLMoE: Open Mixture-of-Experts Language Models","created":"2024-09-03","cited_by_count":301,"area":"moe"},{"id":"2202.01169","title":"Unified Scaling Laws for Routed Language Models","created":"2022-02-02","cited_by_count":253,"area":null},{"id":"2407.06204","title":"A Survey on Mixture of Experts in Large Language Models","created":"2024-06-26","cited_by_count":237,"area":"moe"},{"id":"2212.12017","title":"OPT-IML: Scaling Language Model Instruction Meta Learning through the Lens of Generalization","created":"2022-12-22","cited_by_count":236,"area":null},{"id":"2205.10770","title":"Memorization Without Overfitting: Analyzing the Training Dynamics of Large Language Models","created":"2022-05-22","cited_by_count":220,"area":null},{"id":"2211.15841","title":"MegaBlocks: Efficient Sparse Training with Mixture-of-Experts","created":"2022-11-29","cited_by_count":209,"area":"moe"},{"id":"2212.05055","title":"Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints","created":"2022-12-09","cited_by_count":209,"area":"moe"},{"id":"2308.16369","title":"SARATHI: Efficient LLM Inference by Piggybacking Decodes with Chunked Prefills","created":"2023-08-31","cited_by_count":193,"area":"inference"},{"id":"2208.03306","title":"Branch-Train-Merge: Embarrassingly Parallel Training of Expert Language Models","created":"2022-08-05","cited_by_count":191,"area":null},{"id":"2402.01739","title":"OpenMoE: An Early Effort on Open Mixture-of-Experts Language Models","created":"2024-01-29","cited_by_count":187,"area":"moe"},{"id":"2312.03863","title":"Efficient Large Language Models: A Survey","created":"2023-12-06","cited_by_count":186,"area":null},{"id":"2204.09179","title":"On the Representation Collapse of Sparse Mixture of Experts","created":"2022-04-20","cited_by_count":157,"area":"moe"},{"id":"2308.12284","title":"D4: Improving LLM Pretraining via Document De-Duplication and Diversification","created":"2023-08-23","cited_by_count":150,"area":null},{"id":"2209.01667","title":"A Review of Sparse Expert Models in Deep Learning","created":"2022-09-04","cited_by_count":148,"area":"moe"},{"id":"2201.07520","title":"CM3: A Causal Masked Multimodal Model of the Internet","created":"2022-01-19","cited_by_count":142,"area":null},{"id":"2304.11633","title":"Evaluating ChatGPT's Information Extraction Capabilities: An Assessment of Performance, Explainability, Calibration, and Faithfulness","created":"2023-04-23","cited_by_count":130,"area":null},{"id":"2310.10378","title":"Cross-Lingual Consistency of Factual Knowledge in Multilingual Language Models","created":"2023-10-16","cited_by_count":118,"area":null},{"id":"2209.15611","title":"Protein structure generation via folding diffusion","created":"2022-09-30","cited_by_count":115,"area":"diffusion"},{"id":"2204.10628","title":"Autoregressive Search Engines: Generating Substrings as Document Identifiers","created":"2022-04-22","cited_by_count":113,"area":null},{"id":"2403.08540","title":"Language models scale reliably with over-training and on downstream tasks","created":"2024-03-13","cited_by_count":113,"area":null},{"id":"2409.15790","title":"Small Language Models: Survey, Measurements, and Insights","created":"2024-09-24","cited_by_count":112,"area":null},{"id":"2210.17223","title":"Accelerating Distributed MoE Training and Inference with Lina","created":"2022-10-31","cited_by_count":109,"area":"moe"},{"id":"2310.01334","title":"Merge, Then Compress: Demystify Efficient SMoE with Hints from Its Routing Policy","created":"2023-10-02","cited_by_count":108,"area":"moe"},{"id":"2210.03050","title":"State-of-the-art generalisation research in NLP: A taxonomy and review","created":"2022-10-06","cited_by_count":98,"area":null},{"id":"2403.02181","title":"Not All Layers of LLMs Are Necessary During Inference","created":"2024-03-04","cited_by_count":96,"area":null},{"id":"2212.09535","title":"BLOOM+1: Adding Language Support to BLOOM for Zero-Shot Prompting","created":"2022-12-19","cited_by_count":90,"area":null},{"id":"2302.07027","title":"AdapterSoup: Weight Averaging to Improve Generalization of Pretrained Language Models","created":"2023-02-14","cited_by_count":84,"area":null},{"id":"2302.11529","title":"Modular Deep Learning","created":"2023-02-22","cited_by_count":83,"area":null},{"id":"2212.06713","title":"Structured Prompting: Scaling In-Context Learning to 1,000 Examples","created":"2022-12-13","cited_by_count":81,"area":null},{"id":"2212.09803","title":"Training Trajectories of Language Models Across Scales","created":"2022-12-19","cited_by_count":81,"area":null},{"id":"2206.00277","title":"Task-Specific Expert Pruning for Sparse Mixture-of-Experts","created":"2022-06-01","cited_by_count":72,"area":"moe"},{"id":"2309.14393","title":"LLMCarbon: Modeling the end-to-end Carbon Footprint of Large Language Models","created":"2023-09-25","cited_by_count":72,"area":"moe"},{"id":"2310.14735","title":"Unleashing the potential of prompt engineering for large language models","created":"2023-10-23","cited_by_count":72,"area":null},{"id":"2402.03804","title":"ReLU$^2$ Wins: Discovering Efficient Activation Functions for Sparse LLMs","created":"2024-02-06","cited_by_count":67,"area":null},{"id":"2305.14705","title":"Mixture-of-Experts Meets Instruction Tuning:A Winning Combination for Large Language Models","created":"2023-05-24","cited_by_count":65,"area":"moe"},{"id":"2402.09748","title":"Model Compression and Efficient Inference for Large Language Models: A Survey","created":"2024-02-15","cited_by_count":64,"area":null},{"id":"2407.21770","title":"MoMa: Efficient Early-Fusion Pre-training with Mixture of Modality-Aware Experts","created":"2024-07-31","cited_by_count":62,"area":"moe"},{"id":"2410.12013","title":"MoE-Pruner: Pruning Mixture-of-Experts Large Language Model using the Hints from Its Router","created":"2024-10-15","cited_by_count":62,"area":"moe"},{"id":"2303.14177","title":"Scaling Expert Language Models with Unsupervised Domain Discovery","created":"2023-03-24","cited_by_count":60,"area":null},{"id":"2401.04081","title":"MoE-Mamba: Efficient Selective State Space Models with Mixture of Experts","created":"2024-01-08","cited_by_count":60,"area":"moe"},{"id":"2406.06563","title":"Skywork-MoE: A Deep Dive into Training Techniques for Mixture-of-Experts Language Models","created":"2024-06-03","cited_by_count":59,"area":"moe"},{"id":"2205.01703","title":"Improving In-Context Few-Shot Learning via Self-Supervised Training","created":"2022-05-03","cited_by_count":57,"area":null},{"id":"2407.01531","title":"Sparse Diffusion Policy: A Sparse, Reusable, and Flexible Policy for Robot Learning","created":"2024-07-01","cited_by_count":57,"area":"diffusion"},{"id":"2303.10845","title":"PanGu-{\\Sigma}: Towards Trillion Parameter Language Model with Sparse Heterogeneous Computing","created":"2023-03-20","cited_by_count":56,"area":null},{"id":"2305.19268","title":"Intriguing Properties of Quantization at Scale","created":"2023-05-30","cited_by_count":49,"area":null},{"id":"2303.06318","title":"A Hybrid Tensor-Expert-Data Parallelism Approach to Optimize Mixture-of-Experts Training","created":"2023-03-11","cited_by_count":47,"area":"moe"},{"id":"2412.14219","title":"A Survey on Inference Optimization Techniques for Mixture of Experts Models","created":"2024-12-18","cited_by_count":47,"area":"moe"},{"id":"2309.08520","title":"Scaling Laws for Sparsely-Connected Foundation Models","created":"2023-09-15","cited_by_count":43,"area":null},{"id":"2310.16795","title":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","created":"2023-10-25","cited_by_count":42,"area":"moe"},{"id":"2505.22323","title":"Advancing Expert Specialization for Better MoE","created":"2025-05-28","cited_by_count":39,"area":"moe"},{"id":"2410.07064","title":"Data Selection via Optimal Control for Language Models","created":"2024-10-09","cited_by_count":38,"area":null},{"id":"2205.10034","title":"MoESys: A Distributed and Efficient Mixture-of-Experts Training and Inference System for Internet Services","created":"2022-05-20","cited_by_count":37,"area":"moe"},{"id":"2303.06182","title":"Towards MoE Deployment: Mitigating Inefficiencies in Mixture-of-Expert (MoE) Inference","created":"2023-03-10","cited_by_count":37,"area":"moe"},{"id":"2310.09832","title":"Merging Experts into One: Improving Computational Efficiency of Mixture of Experts","created":"2023-10-15","cited_by_count":37,"area":"moe"},{"id":"2405.03133","title":"Lory: Fully Differentiable Mixture-of-Experts for Autoregressive Language Model Pre-training","created":"2024-05-06","cited_by_count":37,"area":"moe"},{"id":"2307.07982","title":"A Survey of Techniques for Optimizing Transformer Inference","created":"2023-07-16","cited_by_count":35,"area":null},{"id":"2310.02410","title":"Mixture of Quantized Experts (MoQE): Complementary Effect of Low-bit Quantization and Robustness","created":"2023-10-03","cited_by_count":34,"area":"moe"},{"id":"2409.01141","title":"Duplex: A Device for Large Language Models with Mixture of Experts, Grouped Query Attention, and Continuous Batching","created":"2024-09-02","cited_by_count":34,"area":"moe"},{"id":"2401.17671","title":"Contextual Feature Extraction Hierarchies Converge in Large Language Models and the Brain","created":"2024-01-31","cited_by_count":32,"area":null},{"id":"2404.05019","title":"Shortcut-connected Expert Parallelism for Accelerating Mixture-of-Experts","created":"2024-04-07","cited_by_count":32,"area":"moe"},{"id":"2312.06786","title":"Mixture-of-Linear-Experts for Long-term Time Series Forecasting","created":"2023-12-11","cited_by_count":29,"area":null},{"id":"2510.04694","title":"Multilingual Routing in Mixture-of-Experts","created":"2025-10-06","cited_by_count":29,"area":"moe"},{"id":"2401.08383","title":"Exploiting Inter-Layer Expert Affinity for Accelerating Mixture-of-Experts Model Inference","created":"2024-01-16","cited_by_count":28,"area":"moe"},{"id":"2308.15030","title":"SwapMoE: Serving Off-the-shelf MoE-based Large Language Models with Tunable Memory Budget","created":"2023-08-29","cited_by_count":27,"area":"moe"},{"id":"2404.08452","title":"MoE-FFD: Mixture of Experts for Generalized and Parameter-Efficient Face Forgery Detection","created":"2024-04-12","cited_by_count":27,"area":"moe"},{"id":"2402.12656","title":"HyperMoE: Towards Better Mixture of Experts via Transferring Among Experts","created":"2024-02-20","cited_by_count":26,"area":"moe"},{"id":"2201.10890","title":"One Student Knows All Experts Know: From Sparse to Dense","created":"2022-01-26","cited_by_count":25,"area":"moe"},{"id":"2202.07101","title":"A Survey on Dynamic Neural Networks for Natural Language Processing","created":"2022-02-15","cited_by_count":24,"area":null},{"id":"2410.19034","title":"Mixture of Parrots: Experts improve memorization more than reasoning","created":"2024-10-24","cited_by_count":21,"area":"moe"},{"id":"2204.06644","title":"METRO: Efficient Denoising Pretraining of Large Scale Autoencoding Language Models with Model Generated Signals","created":"2022-04-13","cited_by_count":20,"area":null},{"id":"2504.05586","title":"Finding Fantastic Experts in MoEs: A Unified Study for Expert Dropping Strategies and Observations","created":"2025-04-08","cited_by_count":20,"area":"moe"},{"id":"2311.15436","title":"Learning to Skip for Language Modeling","created":"2023-11-26","cited_by_count":19,"area":null},{"id":"2508.01625","title":"EAC-MoE: Expert-Selection Aware Compressor for Mixture-of-Experts Large Language Models","created":"2025-08-03","cited_by_count":18,"area":"moe"},{"id":"2210.12574","title":"The Curious Case of Absolute Position Embeddings","created":"2022-10-23","cited_by_count":17,"area":null},{"id":"2212.08286","title":"ALERT: Adapting Language Models to Reasoning Tasks","created":"2022-12-16","cited_by_count":17,"area":"lr"},{"id":"2504.14960","title":"MoE Parallel Folding: Heterogeneous Parallelism Mappings for Efficient Large-Scale MoE Model Training with Megatron Core","created":"2025-04-21","cited_by_count":17,"area":"moe"},{"id":"2205.11726","title":"On the Role of Bidirectionality in Language Model Pre-Training","created":"2022-05-24","cited_by_count":15,"area":null},{"id":"2409.06669","title":"DA-MoE: Towards Dynamic Expert Allocation for Mixture-of-Experts Models","created":"2024-09-10","cited_by_count":15,"area":"moe"},{"id":"2507.11181","title":"Mixture of Experts in Large Language Models","created":"2025-07-15","cited_by_count":15,"area":"moe"},{"id":"2211.13491","title":"Spatial Mixture-of-Experts","created":"2022-11-24","cited_by_count":14,"area":"moe"},{"id":"2310.02784","title":"MAD Max Beyond Single-Node: Enabling Large Machine Learning Model Acceleration on Distributed Systems","created":"2023-10-04","cited_by_count":14,"area":null},{"id":"2312.06798","title":"Building Trustworthy NeuroSymbolic AI Systems: Consistency, Reliability, Explainability, and Safety","created":"2023-12-05","cited_by_count":14,"area":null},{"id":"2212.03597","title":"DeepSpeed Data Efficiency: Improving Deep Learning Model Quality and Training Efficiency via Efficient Data Sampling and Routing","created":"2022-12-07","cited_by_count":13,"area":null},{"id":"2308.05502","title":"Bringing order into the realm of Transformer-based language models for artificial intelligence and law","created":"2023-08-10","cited_by_count":13,"area":null},{"id":"2205.12399","title":"Sparse Mixers: Combining MoE and Mixing to build a more efficient BERT","created":"2022-05-24","cited_by_count":12,"area":"moe"}],"next_cursor":"100"}