{"total":508,"rows":[{"id":"2305.11627","title":"LLM-Pruner: On the Structural Pruning of Large Language Models","created":"2023-05-19","cited_by_count":650,"area":null},{"id":"2206.01861","title":"ZeroQuant: Efficient and Affordable Post-Training Quantization for Large-Scale Transformers","created":"2022-06-04","cited_by_count":526,"area":null},{"id":"2201.05596","title":"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale","created":"2022-01-14","cited_by_count":404,"area":"moe"},{"id":"2101.02388","title":"Knowledge Distillation in Iterative Generative Models for Improved Sampling Speed","created":"2021-01-07","cited_by_count":348,"area":"diffusion"},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2207.07061","title":"Confident Adaptive Language Modeling","created":"2022-07-14","cited_by_count":310,"area":"inference"},{"id":"2402.13116","title":"A Survey on Knowledge Distillation of Large Language Models","created":"2024-02-20","cited_by_count":284,"area":"distill"},{"id":"2101.01321","title":"I-BERT: Integer-only BERT Quantization","created":"2021-01-05","cited_by_count":255,"area":null},{"id":"2401.06118","title":"Extreme Compression of Large Language Models via Additive Quantization","created":"2024-01-11","cited_by_count":232,"area":null},{"id":"2207.00112","title":"Language model compression with weighted low-rank factorization","created":"2022-06-30","cited_by_count":195,"area":null},{"id":"2108.13323","title":"FedKD: Communication Efficient Federated Learning via Knowledge Distillation","created":"2021-08-30","cited_by_count":187,"area":"distill"},{"id":"2109.04838","title":"Block Pruning For Faster Transformers","created":"2021-09-10","cited_by_count":177,"area":null},{"id":"2204.00408","title":"Structured Pruning Learns Compact and Accurate Models","created":"2022-04-01","cited_by_count":170,"area":null},{"id":"2402.03898","title":"DistiLLM: Towards Streamlined Distillation for Large Language Models","created":"2024-02-06","cited_by_count":163,"area":"distill"},{"id":"2204.09656","title":"A Fast Post-Training Pruning Framework for Transformers","created":"2022-03-29","cited_by_count":162,"area":null},{"id":"2110.01786","title":"MoEfication: Transformer Feed-forward Layers are Mixtures of Experts","created":"2021-10-05","cited_by_count":152,"area":"moe"},{"id":"2107.00910","title":"Learned Token Pruning for Transformers","created":"2021-07-02","cited_by_count":139,"area":"inference"},{"id":"2306.08543","title":"MiniLLM: On-Policy Distillation of Large Language Models","created":"2023-06-14","cited_by_count":139,"area":"distill"},{"id":"2101.04731","title":"SEED: Self-supervised Distillation For Visual Representation","created":"2021-01-12","cited_by_count":134,"area":null},{"id":"2108.05542","title":"AMMUS : A Survey of Transformer-based Pretrained Models in Natural Language Processing","created":"2021-08-12","cited_by_count":131,"area":null},{"id":"2102.00650","title":"Rethinking Soft Labels for Knowledge Distillation: A Bias-Variance Tradeoff Perspective","created":"2021-02-01","cited_by_count":119,"area":"distill"},{"id":"2110.08534","title":"Lifelong Pretraining: Continually Adapting Language Models to Emerging Corpora","created":"2021-10-16","cited_by_count":115,"area":null},{"id":"2401.10491","title":"Knowledge Fusion of Large Language Models","created":"2024-01-19","cited_by_count":115,"area":null},{"id":"2307.15190","title":"f-Divergence Minimization for Sequence-Level Knowledge Distillation","created":"2023-07-27","cited_by_count":114,"area":"distill"},{"id":"2306.11222","title":"LoSparse: Structured Compression of Large Language Models based on Low-Rank and Sparse Approximation","created":"2023-06-20","cited_by_count":104,"area":null},{"id":"2312.15234","title":"Towards Efficient Generative Large Language Model Serving: A Survey from Algorithms to Systems","created":"2023-12-23","cited_by_count":102,"area":"inference"},{"id":"2107.06641","title":"Trustworthy AI: A Computational Perspective","created":"2021-07-12","cited_by_count":93,"area":null},{"id":"2210.01351","title":"Less is More: Task-aware Layer-wise Distillation for Language Model Compression","created":"2022-10-04","cited_by_count":93,"area":null},{"id":"2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","created":"2022-12-28","cited_by_count":81,"area":null},{"id":"2104.08500","title":"Vision Transformer Pruning","created":"2021-04-17","cited_by_count":79,"area":null},{"id":"2402.11295","title":"OneBit: Towards Extremely Low-bit Large Language Models","created":"2024-02-17","cited_by_count":79,"area":null},{"id":"2402.09748","title":"Model Compression and Efficient Inference for Large Language Models: A Survey","created":"2024-02-15","cited_by_count":64,"area":null},{"id":"2104.07163","title":"Annealing Knowledge Distillation","created":"2021-04-14","cited_by_count":61,"area":"distill"},{"id":"2106.04570","title":"BERT Learns to Teach: Knowledge Distillation with Meta Learning","created":"2021-06-08","cited_by_count":61,"area":"distill"},{"id":"2107.01378","title":"Learning Efficient Vision Transformers via Fine-Grained Manifold Distillation","created":"2021-07-03","cited_by_count":60,"area":"distill"},{"id":"2207.03337","title":"Factorizing Knowledge in Neural Networks","created":"2022-07-04","cited_by_count":59,"area":null},{"id":"2401.02038","title":"Understanding LLMs: A Comprehensive Overview from Training to Inference","created":"2024-01-04","cited_by_count":59,"area":null},{"id":"2511.10643","title":"Black-Box On-Policy Distillation of Large Language Models","created":"2025-11-13","cited_by_count":59,"area":"distill"},{"id":"2107.11817","title":"Go Wider Instead of Deeper","created":"2021-07-25","cited_by_count":57,"area":"moe"},{"id":"2411.01016","title":"MoE-I$^2$: Compressing Mixture of Experts Models through Inter-Expert Pruning and Intra-Expert Low-Rank Decomposition","created":"2024-11-01","cited_by_count":57,"area":"moe"},{"id":"2105.14444","title":"NAS-BERT: Task-Agnostic and Adaptive-Size BERT Compression with Neural Architecture Search","created":"2021-05-30","cited_by_count":56,"area":null},{"id":"2210.07795","title":"EfficientVLM: Fast and Accurate Vision-Language Models via Knowledge Distillation and Modal-adaptive Pruning","created":"2022-10-14","cited_by_count":56,"area":"mllm"},{"id":"2407.01885","title":"Survey on Knowledge Distillation for Large Language Models: Methods, Evaluation, and Application","created":"2024-07-02","cited_by_count":56,"area":"distill"},{"id":"2408.07990","title":"FuseChat: Knowledge Fusion of Chat Models","created":"2024-08-15","cited_by_count":55,"area":null},{"id":"2111.05193","title":"A Survey on Green Deep Learning","created":"2021-11-08","cited_by_count":52,"area":null},{"id":"2302.04089","title":"ZipLM: Inference-Aware Structured Pruning of Language Models","created":"2023-02-07","cited_by_count":49,"area":null},{"id":"2102.01386","title":"AutoFreeze: Automatically Freezing Model Blocks to Accelerate Fine-tuning","created":"2021-02-02","cited_by_count":48,"area":null},{"id":"2408.09632","title":"MoDeGPT: Modular Decomposition for Large Language Model Compression","created":"2024-08-19","cited_by_count":48,"area":null},{"id":"2205.15308","title":"Parameter-Efficient and Student-Friendly Knowledge Distillation","created":"2022-05-28","cited_by_count":47,"area":"distill"},{"id":"2112.12731","title":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","created":"2021-12-23","cited_by_count":46,"area":null},{"id":"2402.05964","title":"A Survey on Transformer Compression","created":"2024-02-05","cited_by_count":46,"area":null},{"id":"2202.07105","title":"A Survey on Model Compression and Acceleration for Pretrained Language Models","created":"2022-02-15","cited_by_count":45,"area":null},{"id":"2103.08784","title":"LightningDOT: Pre-training Visual-Semantic Embeddings for Real-Time Image-Text Retrieval","created":"2021-03-16","cited_by_count":44,"area":null},{"id":"2110.07038","title":"Towards Efficient NLP: A Standard Evaluation and A Strong Baseline","created":"2021-10-13","cited_by_count":44,"area":null},{"id":"2202.03680","title":"Exploring Inter-Channel Correlation for Diversity-preserved KnowledgeDistillation","created":"2022-02-08","cited_by_count":44,"area":"distill"},{"id":"2311.07052","title":"Towards the Law of Capacity Gap in Distilling Language Models","created":"2023-11-13","cited_by_count":44,"area":"distill"},{"id":"2105.12967","title":"Selective Knowledge Distillation for Neural Machine Translation","created":"2021-05-27","cited_by_count":43,"area":"distill"},{"id":"2109.05427","title":"Not All Negatives are Equal: Label-Aware Contrastive Loss for Fine-grained Text Classification","created":"2021-09-12","cited_by_count":43,"area":null},{"id":"2106.06168","title":"Generate, Annotate, and Learn: NLP with Synthetic Text","created":"2021-06-11","cited_by_count":42,"area":"distill"},{"id":"2404.08763","title":"CATS: Contextually-Aware Thresholding for Sparsity in Large Language Models","created":"2024-04-12","cited_by_count":41,"area":null},{"id":"2104.08682","title":"Rethinking Network Pruning -- under the Pre-train and Fine-tune Paradigm","created":"2021-04-18","cited_by_count":40,"area":null},{"id":"2105.11618","title":"TR-BERT: Dynamic Token Reduction for Accelerating BERT Inference","created":"2021-05-25","cited_by_count":40,"area":"inference"},{"id":"2106.01023","title":"One Teacher is Enough? Pre-trained Language Model Distillation from Multiple Teachers","created":"2021-06-02","cited_by_count":40,"area":"distill"},{"id":"2109.15082","title":"Towards Efficient Post-training Quantization of Pre-trained Language Models","created":"2021-09-30","cited_by_count":39,"area":null},{"id":"2105.13880","title":"Knowledge Inheritance for Pre-trained Language Models","created":"2021-05-28","cited_by_count":38,"area":null},{"id":"2109.03228","title":"Beyond Preserved Accuracy: Evaluating Loyalty and Robustness of BERT Compression","created":"2021-09-07","cited_by_count":38,"area":null},{"id":"2109.11295","title":"Dynamic Knowledge Distillation for Pre-trained Language Models","created":"2021-09-23","cited_by_count":38,"area":"distill"},{"id":"2302.00219","title":"Knowledge Distillation on Graphs: A Survey","created":"2023-02-01","cited_by_count":38,"area":"distill"},{"id":"2204.07675","title":"MoEBERT: from BERT to Mixture-of-Experts via Importance-Guided Adaptation","created":"2022-04-15","cited_by_count":37,"area":"moe"},{"id":"2212.10670","title":"In-context Learning Distillation: Transferring Few-shot Learning Ability of Pre-trained Language Models","created":"2022-12-20","cited_by_count":37,"area":null},{"id":"2602.20574","title":"GATES: Self-Distillation under Privileged Context with Consensus Gating","created":"2026-02-24","cited_by_count":35,"area":"distill"},{"id":"2209.02432","title":"ViTKD: Practical Guidelines for ViT feature knowledge distillation","created":"2022-09-06","cited_by_count":34,"area":"distill"},{"id":"2109.08359","title":"Distilling Linguistic Context for Language Model Compression","created":"2021-09-17","cited_by_count":33,"area":"distill"},{"id":"2303.03446","title":"Students Parrot Their Teachers: Membership Inference on Model Distillation","created":"2023-03-06","cited_by_count":33,"area":"safety"},{"id":"2108.05698","title":"Preventing Catastrophic Forgetting and Distribution Mismatch in Knowledge Distillation via Synthetic Data","created":"2021-08-11","cited_by_count":32,"area":"distill"},{"id":"2110.15094","title":"Mosaicking to Distill: Knowledge Distillation from Out-of-Domain Data","created":"2021-10-27","cited_by_count":32,"area":"distill"},{"id":"2201.05729","title":"CLIP-TD: CLIP Targeted Distillation for Vision-Language Tasks","created":"2022-01-15","cited_by_count":32,"area":null},{"id":"2203.01670","title":"A Simple Hash-Based Early Exiting Approach For Language Understanding and Generation","created":"2022-03-03","cited_by_count":32,"area":"inference"},{"id":"2107.13686","title":"AutoTinyBERT: Automatic Hyper-parameter Optimization for Efficient Pre-trained Language Models","created":"2021-07-29","cited_by_count":31,"area":null},{"id":"2112.15278","title":"Data-Free Knowledge Transfer: A Survey","created":"2021-12-31","cited_by_count":31,"area":"distill"},{"id":"2506.14728","title":"AgentDistill: Training-Free Agent Distillation with Generalizable MCP Boxes","created":"2025-06-17","cited_by_count":30,"area":"agents"},{"id":"2203.16172","title":"Self-Distillation from the Last Mini-Batch for Consistency Regularization","created":"2022-03-30","cited_by_count":29,"area":"distill"},{"id":"2204.13154","title":"Attention Mechanism in Neural Networks: Where it Comes and Where it Goes","created":"2022-04-27","cited_by_count":29,"area":null},{"id":"2310.18933","title":"Label Poisoning is All You Need","created":"2023-10-29","cited_by_count":29,"area":"safety"},{"id":"2410.17215","title":"MiniPLM: Knowledge Distillation for Pre-Training Language Models","created":"2024-10-22","cited_by_count":29,"area":"distill"},{"id":"2110.08678","title":"Improving Transformers with Probabilistic Attention Keys","created":"2021-10-16","cited_by_count":28,"area":null},{"id":"2203.15935","title":"Graph Neural Networks in IoT: A Survey","created":"2022-03-29","cited_by_count":28,"area":null},{"id":"2310.14747","title":"MCC-KD: Multi-CoT Consistent Knowledge Distillation","created":"2023-10-23","cited_by_count":28,"area":"lr"},{"id":"2105.05912","title":"MATE-KD: Masked Adversarial TExt, a Companion to Knowledge Distillation","created":"2021-05-12","cited_by_count":27,"area":"distill"},{"id":"2110.05323","title":"ProgFed: Effective, Communication, and Computation Efficient Federated Learning by Progressive Training","created":"2021-10-11","cited_by_count":27,"area":null},{"id":"2201.06618","title":"VAQF: Fully Automatic Software-Hardware Co-Design Framework for Low-Bit Vision Transformer","created":"2022-01-17","cited_by_count":27,"area":null},{"id":"2302.06586","title":"Stitchable Neural Networks","created":"2023-02-13","cited_by_count":26,"area":null},{"id":"2303.04185","title":"Gradient-Free Structured Pruning with Unlabeled Data","created":"2023-03-07","cited_by_count":26,"area":null},{"id":"2305.10010","title":"AD-KD: Attribution-Driven Knowledge Distillation for Language Model Compression","created":"2023-05-17","cited_by_count":26,"area":"distill"},{"id":"2308.04268","title":"Teacher-Student Architecture for Knowledge Distillation: A Survey","created":"2023-08-08","cited_by_count":26,"area":"distill"},{"id":"2407.19409","title":"LLAVADI: What Matters For Multimodal Large Language Models Distillation","created":"2024-07-28","cited_by_count":26,"area":"mllm"},{"id":"2302.09632","title":"HomoDistil: Homotopic Task-Agnostic Distillation of Pre-trained Transformers","created":"2023-02-19","cited_by_count":25,"area":"distill"},{"id":"2305.12129","title":"Lifting the Curse of Capacity Gap in Distilling Language Models","created":"2023-05-20","cited_by_count":25,"area":"moe"},{"id":"2402.12842","title":"PromptKD: Distilling Student-Friendly Knowledge for Generative Language Models via Prompt Tuning","created":"2024-02-20","cited_by_count":25,"area":"distill"},{"id":"2402.13720","title":"Ouroboros: Generating Longer Drafts Phrase by Phrase for Faster Speculative Decoding","created":"2024-02-21","cited_by_count":25,"area":"inference"}],"next_cursor":"100"}