{"total":110,"rows":[{"id":"2402.03216","title":"M3-Embedding: Multi-Linguality, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation","created":"2024-02-05","cited_by_count":1073,"area":"distill"},{"id":"2402.13116","title":"A Survey on Knowledge Distillation of Large Language Models","created":"2024-02-20","cited_by_count":284,"area":"distill"},{"id":"2402.03898","title":"DistiLLM: Towards Streamlined Distillation for Large Language Models","created":"2024-02-06","cited_by_count":163,"area":"distill"},{"id":"2212.08320","title":"Autoencoders as Cross-Modal Teachers: Can Pretrained 2D Image Transformers Help 3D Representation Learning?","created":"2022-12-16","cited_by_count":122,"area":"distill"},{"id":"2307.15190","title":"f-Divergence Minimization for Sequence-Level Knowledge Distillation","created":"2023-07-27","cited_by_count":114,"area":"distill"},{"id":"2104.02096","title":"Compressing Visual-linguistic Model via Knowledge Distillation","created":"2021-04-05","cited_by_count":86,"area":"distill"},{"id":"2308.02019","title":"Baby Llama: knowledge distillation from an ensemble of teachers trained on a small dataset with no performance penalty","created":"2023-08-03","cited_by_count":57,"area":"distill"},{"id":"2311.07052","title":"Towards the Law of Capacity Gap in Distilling Language Models","created":"2023-11-13","cited_by_count":44,"area":"distill"},{"id":"2106.06168","title":"Generate, Annotate, and Learn: NLP with Synthetic Text","created":"2021-06-11","cited_by_count":42,"area":"distill"},{"id":"2109.08359","title":"Distilling Linguistic Context for Language Model Compression","created":"2021-09-17","cited_by_count":33,"area":"distill"},{"id":"2208.10160","title":"PANDA: Prompt Transfer Meets Knowledge Distillation for Efficient Model Adaptation","created":"2022-08-22","cited_by_count":31,"area":"distill"},{"id":"2305.08776","title":"Bridging the Domain Gap: Self-Supervised 3D Scene Understanding with Foundation Models","created":"2023-05-15","cited_by_count":27,"area":"distill"},{"id":"2302.09632","title":"HomoDistil: Homotopic Task-Agnostic Distillation of Pre-trained Transformers","created":"2023-02-19","cited_by_count":25,"area":"distill"},{"id":"2305.05010","title":"Do Not Blindly Imitate the Teacher: Using Perturbed Loss for Knowledge Distillation","created":"2023-05-08","cited_by_count":25,"area":"distill"},{"id":"2502.11191","title":"Primus: A Pioneering Collection of Open-Source Datasets for Cybersecurity LLM Training","created":"2025-02-16","cited_by_count":23,"area":"distill"},{"id":"2107.02681","title":"VidLanKD: Improving Language Understanding via Video-Distilled Knowledge Transfer","created":"2021-07-06","cited_by_count":22,"area":"distill"},{"id":"2401.04810","title":"Translate-Distill: Learning Cross-Language Dense Retrieval by Translation and Distillation","created":"2024-01-09","cited_by_count":22,"area":"distill"},{"id":"2206.01859","title":"Extreme Compression for Pre-trained Transformers Made Simple and Efficient","created":"2022-06-04","cited_by_count":19,"area":"distill"},{"id":"2411.18674","title":"Active Data Curation Effectively Distills Large-Scale Multimodal Models","created":"2024-11-27","cited_by_count":19,"area":"distill"},{"id":"2305.02031","title":"A Systematic Study of Knowledge Distillation for Natural Language Generation with Pseudo-Target Training","created":"2023-05-03","cited_by_count":16,"area":"distill"},{"id":"2206.07808","title":"Alexa Teacher Model: Pretraining and Distilling Multi-Billion-Parameter Encoders for Natural Language Understanding Systems","created":"2022-06-15","cited_by_count":15,"area":"distill"},{"id":"2112.07074","title":"Towards a Unified Foundation Model: Jointly Pre-Training Transformers on Unpaired Images and Text","created":"2021-12-14","cited_by_count":14,"area":"distill"},{"id":"2406.05322","title":"Teaching-Assistant-in-the-Loop: Improving Knowledge Distillation from Imperfect Teacher Models in Low-Budget Scenarios","created":"2024-06-08","cited_by_count":12,"area":"distill"},{"id":"2211.11014","title":"Understanding and Improving Knowledge Distillation for Quantization-Aware Training of Large Transformer Encoders","created":"2022-11-20","cited_by_count":11,"area":"distill"},{"id":"2301.12005","title":"EmbedDistill: A Geometric Knowledge Distillation for Information Retrieval","created":"2023-01-27","cited_by_count":11,"area":"distill"},{"id":"2305.15781","title":"VanillaKD: Revisit the Power of Vanilla Knowledge Distillation from Small Scale to Large Scale","created":"2023-05-25","cited_by_count":11,"area":"distill"},{"id":"2406.02886","title":"PLaD: Preference-based Large Language Model Distillation with Pseudo-Preference Pairs","created":"2024-06-05","cited_by_count":11,"area":"distill"},{"id":"2101.08890","title":"Distilling Large Language Models into Tiny and Effective Students using pQRNN","created":"2021-01-21","cited_by_count":10,"area":"distill"},{"id":"2311.08640","title":"Multistage Collaborative Knowledge Distillation from a Large Language Model for Semi-Supervised Sequence Generation","created":"2023-11-15","cited_by_count":10,"area":"distill"},{"id":"2502.11306","title":"Smoothing Out Hallucinations: Mitigating LLM Hallucination with Smoothed Knowledge Distillation","created":"2025-02-16","cited_by_count":10,"area":"distill"},{"id":"2212.05956","title":"Improving Generalization of Pre-trained Language Models via Stochastic Weight Averaging","created":"2022-12-12","cited_by_count":9,"area":"distill"},{"id":"2408.10210","title":"A Survey on Symbolic Knowledge Distillation of Large Language Models","created":"2024-07-12","cited_by_count":9,"area":"distill"},{"id":"2410.14609","title":"DiSCo: LLM Knowledge Distillation for Efficient Sparse Retrieval in Conversational Search","created":"2024-10-18","cited_by_count":9,"area":"distill"},{"id":"2508.15709","title":"Position Bias Mitigates Position Bias:Mitigate Position Bias Through Inter-Position Knowledge Distillation","created":"2025-08-21","cited_by_count":9,"area":"distill"},{"id":"2310.13760","title":"Enhancing Abstractiveness of Summarization Models through Calibrated Distillation","created":"2023-10-20","cited_by_count":8,"area":"distill"},{"id":"2312.05979","title":"NovaCOMET: Open Commonsense Foundation Models with Symbolic Knowledge Distillation","created":"2023-12-10","cited_by_count":8,"area":"distill"},{"id":"2403.13590","title":"Teacher-Student Training for Debiasing: General Permutation Debiasing for Large Language Models","created":"2024-03-20","cited_by_count":8,"area":"distill"},{"id":"2305.05027","title":"Web Content Filtering through knowledge distillation of Large Language Models","created":"2023-05-08","cited_by_count":7,"area":"distill"},{"id":"2104.11928","title":"Extract then Distill: Efficient and Effective Task-Agnostic BERT Distillation","created":"2021-04-24","cited_by_count":6,"area":"distill"},{"id":"2309.15494","title":"VideoAdviser: Video Knowledge Distillation for Multimodal Transfer Learning","created":"2023-09-27","cited_by_count":6,"area":"distill"},{"id":"2403.19754","title":"GOLD: Generalized Knowledge Distillation via Out-of-Distribution-Guided Language Data Generation","created":"2024-03-28","cited_by_count":6,"area":"distill"},{"id":"2410.18097","title":"RRADistill: Distilling LLMs' Passage Ranking Ability for Long-Tail Queries Document Re-Ranking on a Search Engine","created":"2024-10-08","cited_by_count":6,"area":"distill"},{"id":"2412.15255","title":"Data Laundering: Artificially Boosting Benchmark Results through Knowledge Distillation","created":"2024-12-15","cited_by_count":6,"area":"distill"},{"id":"2504.13825","title":"Feature Alignment and Representation Transfer in Knowledge Distillation for Large Language Models","created":"2025-04-18","cited_by_count":6,"area":"distill"},{"id":"2507.08336","title":"Distillation versus Contrastive Learning: How to Train Your Rerankers","created":"2025-07-11","cited_by_count":6,"area":"distill"},{"id":"2201.00558","title":"Which Student is Best? A Comprehensive Knowledge Distillation Exam for Task-Specific BERT Models","created":"2022-01-03","cited_by_count":5,"area":"distill"},{"id":"2302.01530","title":"Revisiting Intermediate Layer Distillation for Compressing Language Models: An Overfitting Perspective","created":"2023-02-03","cited_by_count":5,"area":"distill"},{"id":"2306.08909","title":"Bridging the Gap between Decision and Logits in Decision-based Knowledge Distillation for Pre-trained Language Models","created":"2023-06-15","cited_by_count":5,"area":"distill"},{"id":"2404.01353","title":"Efficiently Distilling LLMs for Edge Applications","created":"2024-04-01","cited_by_count":5,"area":"distill"},{"id":"2411.06448","title":"Over-parameterized Student Model via Tensor Decomposition Boosted Knowledge Distillation","created":"2024-11-10","cited_by_count":5,"area":"distill"},{"id":"2502.04499","title":"Revisiting Intermediate-Layer Matching in Knowledge Distillation: Layer-Selection Strategy Doesn't Matter (Much)","created":"2025-02-06","cited_by_count":5,"area":"distill"},{"id":"2109.05696","title":"How to Select One Among All? An Extensive Empirical Study Towards the Robustness of Knowledge Distillation in Natural Language Understanding","created":"2021-09-13","cited_by_count":4,"area":"distill"},{"id":"2204.07674","title":"CILDA: Contrastive Data Augmentation using Intermediate Layer Knowledge Distillation","created":"2022-04-15","cited_by_count":4,"area":"distill"},{"id":"2210.04834","title":"Knowledge Distillation Transfer Sets and their Impact on Downstream NLU Tasks","created":"2022-10-10","cited_by_count":4,"area":"distill"},{"id":"2305.12074","title":"DisCo: Distilled Student Models Co-training for Semi-supervised Text Mining","created":"2023-05-20","cited_by_count":4,"area":"distill"},{"id":"2306.02379","title":"Modular Transformers: Compressing Transformers into Modularized Layers for Flexible Efficient Inference","created":"2023-06-04","cited_by_count":4,"area":"distill"},{"id":"2406.02959","title":"Adversarial Moment-Matching Distillation of Large Language Models","created":"2024-06-05","cited_by_count":4,"area":"distill"},{"id":"2406.16524","title":"The Privileged Students: On the Value of Initialization in Multilingual Knowledge Distillation","created":"2024-06-24","cited_by_count":4,"area":"distill"},{"id":"2409.12586","title":"Efficient Knowledge Distillation: Empowering Small Language Models with Teacher Model Insights","created":"2024-09-19","cited_by_count":4,"area":"distill"},{"id":"2508.02929","title":"Realizing Scaling Laws in Recommender Systems: A Foundation-Expert Paradigm for Hyperscale Model Deployment","created":"2025-08-04","cited_by_count":4,"area":"distill"},{"id":"2303.03600","title":"Adaptive Knowledge Distillation between Text and Speech Pre-trained Models","created":"2023-03-07","cited_by_count":3,"area":"distill"},{"id":"2404.12596","title":"Parameter Efficient Diverse Paraphrase Generation Using Sequence-Level Knowledge Distillation","created":"2024-04-19","cited_by_count":3,"area":"distill"},{"id":"2408.00205","title":"Sentence-wise Speech Summarization: Task, Datasets, and End-to-End Modeling with LM Knowledge Distillation","created":"2024-08-01","cited_by_count":3,"area":"distill"},{"id":"2409.12500","title":"LLMR: Knowledge Distillation with a Large Language Model-Induced Reward","created":"2024-09-19","cited_by_count":3,"area":"distill"},{"id":"2410.03061","title":"DocKD: Knowledge Distillation from LLMs for Open-World Document Understanding Models","created":"2024-10-04","cited_by_count":3,"area":"distill"},{"id":"2504.06036","title":"Multi-Sense Embeddings for Language Models and Knowledge Distillation","created":"2025-04-08","cited_by_count":3,"area":"distill"},{"id":"2504.19024","title":"KETCHUP: K-Step Return Estimation for Sequential Knowledge Distillation","created":"2025-04-26","cited_by_count":3,"area":"distill"},{"id":"2605.23857","title":"Strong Teacher Not Needed? On Distillation in LLM Pretraining","created":"2026-05-22","cited_by_count":3,"area":"distill"},{"id":"2112.05638","title":"DistilCSE: Effective Knowledge Distillation For Contrastive Sentence Embeddings","created":"2021-12-10","cited_by_count":2,"area":"distill"},{"id":"2210.11768","title":"Augmentation with Projection: Towards an Effective and Efficient Data Augmentation Paradigm for Distillation","created":"2022-10-21","cited_by_count":2,"area":"distill"},{"id":"2305.12565","title":"Understanding the Effect of Data Augmentation on Knowledge Distillation","created":"2023-05-21","cited_by_count":2,"area":"distill"},{"id":"2305.18419","title":"Semantic Segmentation with Bidirectional Language Models Improves Long-form ASR","created":"2023-05-28","cited_by_count":2,"area":"distill"},{"id":"2310.17644","title":"torchdistill Meets Hugging Face Libraries for Reproducible, Coding-Free Deep Learning Studies: A Case Study on NLP","created":"2023-10-26","cited_by_count":2,"area":"distill"},{"id":"2403.15886","title":"Leveraging Zero-Shot Prompting for Efficient Language Model Distillation","created":"2024-03-23","cited_by_count":2,"area":"distill"},{"id":"2410.11371","title":"Learning from Imperfect Data: Towards Efficient Knowledge Distillation of Autoregressive Language Models for Text-to-SQL","created":"2024-10-15","cited_by_count":2,"area":"distill"},{"id":"2504.16056","title":"Honey, I Shrunk the Language Model: Impact of Knowledge Distillation Methods on Performance and Explainability","created":"2025-04-22","cited_by_count":2,"area":"distill"},{"id":"2508.15423","title":"An Empirical Study of Knowledge Distillation for Code Understanding Tasks","created":"2025-08-21","cited_by_count":2,"area":"distill"},{"id":"2604.03192","title":"Reliability Gated Multi-Teacher Distillation for Low Resource Abstractive Summarization","created":"2026-04-03","cited_by_count":2,"area":"distill"},{"id":"2109.10147","title":"Knowledge Distillation with Noisy Labels for Natural Language Understanding","created":"2021-09-21","cited_by_count":1,"area":"distill"},{"id":"2209.05869","title":"Multi-stage Distillation Framework for Cross-Lingual Semantic Similarity Matching","created":"2022-09-13","cited_by_count":1,"area":"distill"},{"id":"2311.01689","title":"Data-Free Distillation of Language Model by Text-to-Text Transfer","created":"2023-11-03","cited_by_count":1,"area":"distill"},{"id":"2402.14035","title":"Wisdom of Committee: Diverse Distillation from Large Foundation Models and Domain Experts","created":"2024-02-21","cited_by_count":1,"area":"distill"},{"id":"2409.18565","title":"Harmonizing knowledge Transfer in Neural Network with Unified Distillation","created":"2024-09-27","cited_by_count":1,"area":"distill"},{"id":"2410.13918","title":"FTSmartAudit: A Knowledge Distillation-Enhanced Framework for Automated Smart Contract Auditing Using Fine-Tuned LLMs","created":"2024-10-17","cited_by_count":1,"area":"distill"},{"id":"2412.15303","title":"Self-Evolution Knowledge Distillation for LLM-based Machine Translation","created":"2024-12-19","cited_by_count":1,"area":"distill"},{"id":"2501.16734","title":"Distilling Large Language Models for Network Active Queue Management","created":"2025-01-28","cited_by_count":1,"area":"distill"},{"id":"2506.12038","title":"LCD: Advancing Extreme Low-Bit Clustering for Large Language Models via Knowledge Distillation","created":"2025-05-23","cited_by_count":1,"area":"distill"},{"id":"2510.06502","title":"GUIDE: Guided Initialization and Distillation of Embeddings","created":"2025-10-07","cited_by_count":1,"area":"distill"},{"id":"2602.09509","title":"Beyond Student: An Asymmetric Network for Neural Network Inheritance","created":"2026-02-10","cited_by_count":1,"area":"distill"},{"id":"2602.12674","title":"$\\mathcal{X}$-KD: General Experiential Knowledge Distillation for Large Language Models","created":"2026-02-13","cited_by_count":1,"area":"distill"},{"id":"2309.02887","title":"A deep Natural Language Inference predictor without language-specific training data","created":"2023-09-06","cited_by_count":0,"area":"distill"},{"id":"2310.02549","title":"Heterogeneous Federated Learning Using Knowledge Codistillation","created":"2023-10-04","cited_by_count":0,"area":"distill"},{"id":"2310.13418","title":"GenDistiller: Distilling Pre-trained Language Models based on Generative Models","created":"2023-10-20","cited_by_count":0,"area":"distill"},{"id":"2312.10185","title":"Student as an Inherent Denoiser of Noisy Teacher","created":"2023-12-15","cited_by_count":0,"area":"distill"},{"id":"2403.14494","title":"Learning to Project for Cross-Task Knowledge Distillation","created":"2024-03-21","cited_by_count":0,"area":"distill"},{"id":"2409.12599","title":"Enhancing SLM via ChatGPT and Dataset Augmentation","created":"2024-09-19","cited_by_count":0,"area":"distill"},{"id":"2409.14810","title":"Pre-trained Language Model and Knowledge Distillation for Lightweight Sequential Recommendation","created":"2024-09-23","cited_by_count":0,"area":"distill"},{"id":"2412.18524","title":"HTR-JAND: Handwritten Text Recognition with Joint Attention Network and Knowledge Distillation","created":"2024-12-24","cited_by_count":0,"area":"distill"},{"id":"2501.03670","title":"A Diversity-Enhanced Knowledge Distillation Model for Practical Math Word Problem Solving","created":"2025-01-07","cited_by_count":0,"area":"distill"},{"id":"2503.12608","title":"UniBERT: Adversarial Training for Language-Universal Representations","created":"2025-03-16","cited_by_count":0,"area":"distill"}],"next_cursor":"100"}