{"total":54,"rows":[{"id":"2110.07178","title":"Symbolic Knowledge Distillation: from General Language Models to Commonsense Models","created":"2021-10-14","cited_by_count":270,"area":"distill"},{"id":"2102.00650","title":"Rethinking Soft Labels for Knowledge Distillation: A Bias-Variance Tradeoff Perspective","created":"2021-02-01","cited_by_count":119,"area":"distill"},{"id":"2104.07163","title":"Annealing Knowledge Distillation","created":"2021-04-14","cited_by_count":61,"area":"distill"},{"id":"2106.04570","title":"BERT Learns to Teach: Knowledge Distillation with Meta Learning","created":"2021-06-08","cited_by_count":61,"area":"distill"},{"id":"2109.11295","title":"Dynamic Knowledge Distillation for Pre-trained Language Models","created":"2021-09-23","cited_by_count":38,"area":"distill"},{"id":"2305.10010","title":"AD-KD: Attribution-Driven Knowledge Distillation for Language Model Compression","created":"2023-05-17","cited_by_count":26,"area":"distill"},{"id":"2302.09632","title":"HomoDistil: Homotopic Task-Agnostic Distillation of Pre-trained Transformers","created":"2023-02-19","cited_by_count":25,"area":"distill"},{"id":"2305.05010","title":"Do Not Blindly Imitate the Teacher: Using Perturbed Loss for Knowledge Distillation","created":"2023-05-08","cited_by_count":25,"area":"distill"},{"id":"2104.08945","title":"Data-Efficient Language-Supervised Zero-Shot Learning with Self-Distillation","created":"2021-04-18","cited_by_count":22,"area":"distill"},{"id":"2107.02681","title":"VidLanKD: Improving Language Understanding via Video-Distilled Knowledge Transfer","created":"2021-07-06","cited_by_count":22,"area":"distill"},{"id":"2206.01859","title":"Extreme Compression for Pre-trained Transformers Made Simple and Efficient","created":"2022-06-04","cited_by_count":19,"area":"distill"},{"id":"2301.12923","title":"On student-teacher deviations in distillation: does it pay to disobey?","created":"2023-01-30","cited_by_count":16,"area":"distill"},{"id":"2109.07222","title":"EfficientBERT: Progressively Searching Multilayer Perceptron via Warm-up Knowledge Distillation","created":"2021-09-15","cited_by_count":15,"area":"distill"},{"id":"2402.17110","title":"Sinkhorn Distance Minimization for Knowledge Distillation","created":"2024-02-27","cited_by_count":15,"area":"distill"},{"id":"2302.00444","title":"Improved Knowledge Distillation for Pre-trained Language Models via Knowledge Selection","created":"2023-02-01","cited_by_count":12,"area":"distill"},{"id":"2105.14839","title":"Greedy-layer Pruning: Speeding up Transformer Models for Natural Language Processing","created":"2021-05-31","cited_by_count":11,"area":"distill"},{"id":"2107.09305","title":"Follow Your Path: a Progressive Method for Knowledge Distillation","created":"2021-07-20","cited_by_count":10,"area":"distill"},{"id":"2305.18239","title":"A Study on Knowledge Distillation from Weak Teacher for Scaling Up Pre-trained Language Models","created":"2023-05-26","cited_by_count":9,"area":"distill"},{"id":"2406.12739","title":"Self-Distillation for Model Stacking Unlocks Cross-Lingual NLU in 200+ Languages","created":"2024-06-18","cited_by_count":9,"area":"distill"},{"id":"2110.02432","title":"KNOT: Knowledge Distillation using Optimal Transport for Solving NLP Tasks","created":"2021-10-06","cited_by_count":8,"area":"distill"},{"id":"2305.09098","title":"Weight-Inherited Distillation for Task-Agnostic BERT Compression","created":"2023-05-16","cited_by_count":8,"area":"distill"},{"id":"2305.09651","title":"Tailoring Instructions to Student's Learning Levels Boosts Knowledge Distillation","created":"2023-05-16","cited_by_count":8,"area":"distill"},{"id":"2305.13067","title":"Distilling Robustness into Natural Language Inference Models with Domain-Targeted Augmentation","created":"2023-05-22","cited_by_count":7,"area":"distill"},{"id":"2204.08922","title":"Feature Structure Distillation with Centered Kernel Alignment in BERT Transferring","created":"2022-04-01","cited_by_count":6,"area":"distill"},{"id":"2302.11187","title":"Debiased Distillation by Transplanting the Last Layer","created":"2023-02-22","cited_by_count":6,"area":"distill"},{"id":"2203.09391","title":"When Chosen Wisely, More Data Is What You Need: A Universal Sample-Efficient Strategy For Data Augmentation","created":"2022-03-17","cited_by_count":5,"area":"distill"},{"id":"2204.00548","title":"Unified and Effective Ensemble Knowledge Distillation","created":"2022-04-01","cited_by_count":5,"area":"distill"},{"id":"2211.01071","title":"Gradient Knowledge Distillation for Pre-trained Language Models","created":"2022-11-02","cited_by_count":5,"area":"distill"},{"id":"2306.08909","title":"Bridging the Gap between Decision and Logits in Decision-based Knowledge Distillation for Pre-trained Language Models","created":"2023-06-15","cited_by_count":5,"area":"distill"},{"id":"2502.04499","title":"Revisiting Intermediate-Layer Matching in Knowledge Distillation: Layer-Selection Strategy Doesn't Matter (Much)","created":"2025-02-06","cited_by_count":5,"area":"distill"},{"id":"2412.13243","title":"In-Context Learning Distillation for Efficient Few-Shot Fine-Tuning","created":"2024-12-17","cited_by_count":4,"area":"distill"},{"id":"2110.08551","title":"HRKD: Hierarchical Relational Knowledge Distillation for Cross-domain Language Model Compression","created":"2021-10-16","cited_by_count":3,"area":"distill"},{"id":"2201.12507","title":"AutoDistil: Few-shot Task-agnostic Neural Architecture Search for Distilling Large Language Models","created":"2022-01-29","cited_by_count":3,"area":"distill"},{"id":"2211.01200","title":"Multi-level Distillation of Semantic Knowledge for Pre-training Multilingual Language Model","created":"2022-11-02","cited_by_count":3,"area":"distill"},{"id":"2301.03416","title":"ERNIE 3.0 Tiny: Frustratingly Simple Method to Improve Task-Agnostic Distillation Generalization","created":"2023-01-09","cited_by_count":3,"area":"distill"},{"id":"2301.12609","title":"Knowledge Distillation $\\approx$ Label Smoothing: Fact or Fallacy?","created":"2023-01-30","cited_by_count":3,"area":"distill"},{"id":"2411.00052","title":"Larger models yield better results? Streamlined severity classification of ADHD-related concerns using BERT-based knowledge distillation","created":"2024-10-30","cited_by_count":3,"area":"distill"},{"id":"2101.08106","title":"Learning to Augment for Data-Scarce Domain BERT Knowledge Distillation","created":"2021-01-20","cited_by_count":2,"area":"distill"},{"id":"2109.04641","title":"Learning to Teach with Student Feedback","created":"2021-09-10","cited_by_count":2,"area":"distill"},{"id":"2305.12565","title":"Understanding the Effect of Data Augmentation on Knowledge Distillation","created":"2023-05-21","cited_by_count":2,"area":"distill"},{"id":"2310.17644","title":"torchdistill Meets Hugging Face Libraries for Reproducible, Coding-Free Deep Learning Studies: A Case Study on NLP","created":"2023-10-26","cited_by_count":2,"area":"distill"},{"id":"2510.26038","title":"Do Students Debias Like Teachers? On the Distillability of Bias Mitigation Methods","created":"2025-10-30","cited_by_count":2,"area":"distill"},{"id":"2512.21288","title":"Model Merging via Multi-Teacher Knowledge Distillation","created":"2025-12-24","cited_by_count":2,"area":"distill"},{"id":"2305.07928","title":"AMTSS: An Adaptive Multi-Teacher Single-Student Knowledge Distillation Framework For Multilingual Language Inference","created":"2023-05-13","cited_by_count":1,"area":"distill"},{"id":"2404.19319","title":"Knowledge Distillation vs. Pretraining from Scratch under a Fixed (Computation) Budget","created":"2024-04-30","cited_by_count":1,"area":"distill"},{"id":"2406.09719","title":"Self-Knowledge Distillation for Learning Ambiguity","created":"2024-06-14","cited_by_count":1,"area":"distill"},{"id":"2407.02775","title":"MLKD-BERT: Multi-level Knowledge Distillation for Pre-trained Language Models","created":"2024-07-03","cited_by_count":1,"area":"distill"},{"id":"2309.02887","title":"A deep Natural Language Inference predictor without language-specific training data","created":"2023-09-06","cited_by_count":0,"area":"distill"},{"id":"2401.06356","title":"An Empirical Investigation into the Effect of Parameter Choices in Knowledge Distillation","created":"2024-01-12","cited_by_count":0,"area":"distill"},{"id":"2411.16991","title":"Dynamic Self-Distillation via Previous Mini-batches for Fine-tuning Small Language Models","created":"2024-11-25","cited_by_count":0,"area":"distill"},{"id":"2501.02705","title":"Knowledge Distillation with Adapted Weight","created":"2025-01-06","cited_by_count":0,"area":"distill"},{"id":"2505.01070","title":"Improving Group Fairness in Knowledge Distillation via Laplace Approximation of Early Exits","created":"2025-05-02","cited_by_count":0,"area":"distill"},{"id":"2604.03110","title":"Multi-Aspect Knowledge Distillation for Language Model with Low-rank Factorization","created":"2026-04-03","cited_by_count":0,"area":"distill"},{"id":"2609.22566","title":"Do Student LLMs Inherit OOD Robustness? Invariance-Weighted Distillation for Reliable Knowledge Transfer","created":"2026-09-18","cited_by_count":0,"area":"distill"}],"next_cursor":null}