{"total":410,"rows":[{"id":"2104.13921","title":"Open-vocabulary Object Detection via Vision and Language Knowledge Distillation","created":"2021-04-28","cited_by_count":906,"area":"distill"},{"id":"2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","created":"2021-11-15","cited_by_count":903,"area":"distill"},{"id":"2108.13323","title":"FedKD: Communication Efficient Federated Learning via Knowledge Distillation","created":"2021-08-30","cited_by_count":187,"area":"distill"},{"id":"2202.11539","title":"Self-Supervised Transformers for Unsupervised Object Discovery using Normalized Cut","created":"2022-02-23","cited_by_count":167,"area":"distill"},{"id":"2212.08320","title":"Autoencoders as Cross-Modal Teachers: Can Pretrained 2D Image Transformers Help 3D Representation Learning?","created":"2022-12-16","cited_by_count":122,"area":"distill"},{"id":"2212.04500","title":"Masked Video Distillation: Rethinking Masked Feature Modeling for Self-supervised Video Representation Learning","created":"2022-12-08","cited_by_count":90,"area":"distill"},{"id":"2311.00430","title":"Distil-Whisper: Robust Knowledge Distillation via Large-Scale Pseudo Labelling","created":"2023-11-01","cited_by_count":90,"area":"distill"},{"id":"2104.02096","title":"Compressing Visual-linguistic Model via Knowledge Distillation","created":"2021-04-05","cited_by_count":86,"area":"distill"},{"id":"2303.13802","title":"Decoupled Multimodal Distilling for Emotion Recognition","created":"2023-03-24","cited_by_count":76,"area":"distill"},{"id":"2204.12997","title":"DearKD: Data-Efficient Early Knowledge Distillation for Vision Transformers","created":"2022-04-27","cited_by_count":65,"area":"distill"},{"id":"2203.00048","title":"Multi-modal Alignment using Representation Codebook","created":"2022-02-28","cited_by_count":63,"area":"distill"},{"id":"2104.07163","title":"Annealing Knowledge Distillation","created":"2021-04-14","cited_by_count":61,"area":"distill"},{"id":"2106.04570","title":"BERT Learns to Teach: Knowledge Distillation with Meta Learning","created":"2021-06-08","cited_by_count":61,"area":"distill"},{"id":"2107.01378","title":"Learning Efficient Vision Transformers via Fine-Grained Manifold Distillation","created":"2021-07-03","cited_by_count":60,"area":"distill"},{"id":"2105.01029","title":"Initialization and Regularization of Factorized Neural Layers","created":"2021-05-03","cited_by_count":58,"area":"distill"},{"id":"2106.04928","title":"Reliable Adversarial Distillation with Unreliable Teachers","created":"2021-06-09","cited_by_count":56,"area":"distill"},{"id":"2407.01885","title":"Survey on Knowledge Distillation for Large Language Models: Methods, Evaluation, and Application","created":"2024-07-02","cited_by_count":56,"area":"distill"},{"id":"2205.12654","title":"Bitext Mining Using Distilled Sentence Representations for Low-Resource Languages","created":"2022-05-25","cited_by_count":48,"area":"distill"},{"id":"2205.15308","title":"Parameter-Efficient and Student-Friendly Knowledge Distillation","created":"2022-05-28","cited_by_count":47,"area":"distill"},{"id":"2209.03917","title":"Exploring Target Representations for Masked Autoencoders","created":"2022-09-08","cited_by_count":46,"area":"distill"},{"id":"2311.07052","title":"Towards the Law of Capacity Gap in Distilling Language Models","created":"2023-11-13","cited_by_count":44,"area":"distill"},{"id":"2403.16002","title":"SDSTrack: Self-Distillation Symmetric Adapter Learning for Multi-Modal Visual Object Tracking","created":"2024-03-24","cited_by_count":44,"area":"distill"},{"id":"2206.06067","title":"Better Teacher Better Student: Dynamic Prior Knowledge for Knowledge Distillation","created":"2022-06-13","cited_by_count":41,"area":"distill"},{"id":"2211.14241","title":"Look Around and Refer: 2D Synthetic Semantics Knowledge Distillation for 3D Visual Grounding","created":"2022-11-25","cited_by_count":41,"area":"distill"},{"id":"2503.20083","title":"Universal Cross-Tokenizer Distillation via Approximate Likelihood Matching","created":"2025-03-25","cited_by_count":41,"area":"distill"},{"id":"2106.01023","title":"One Teacher is Enough? Pre-trained Language Model Distillation from Multiple Teachers","created":"2021-06-02","cited_by_count":40,"area":"distill"},{"id":"2201.00007","title":"Confidence-Aware Multi-Teacher Knowledge Distillation","created":"2021-12-30","cited_by_count":39,"area":"distill"},{"id":"2203.10900","title":"Document-Level Relation Extraction with Adaptive Focal Loss and Knowledge Distillation","created":"2022-03-21","cited_by_count":39,"area":"distill"},{"id":"2109.11295","title":"Dynamic Knowledge Distillation for Pre-trained Language Models","created":"2021-09-23","cited_by_count":38,"area":"distill"},{"id":"2305.10005","title":"DinoSR: Self-Distillation and Online Clustering for Self-supervised Speech Representation Learning","created":"2023-05-17","cited_by_count":37,"area":"distill"},{"id":"2310.20494","title":"A Transformer-Based Model With Self-Distillation for Multimodal Emotion Recognition in Conversations","created":"2023-10-31","cited_by_count":37,"area":"distill"},{"id":"2208.12256","title":"Masked Autoencoders Enable Efficient Knowledge Distillers","created":"2022-08-25","cited_by_count":36,"area":"distill"},{"id":"2404.02117","title":"Pre-trained Vision and Language Transformers Are Few-Shot Incremental Learners","created":"2024-04-02","cited_by_count":36,"area":"distill"},{"id":"2106.00903","title":"Rejuvenating Low-Frequency Words: Making the Most of Parallel Data in Non-Autoregressive Translation","created":"2021-06-02","cited_by_count":34,"area":"distill"},{"id":"2209.02432","title":"ViTKD: Practical Guidelines for ViT feature knowledge distillation","created":"2022-09-06","cited_by_count":34,"area":"distill"},{"id":"2109.08359","title":"Distilling Linguistic Context for Language Model Compression","created":"2021-09-17","cited_by_count":33,"area":"distill"},{"id":"2112.15278","title":"Data-Free Knowledge Transfer: A Survey","created":"2021-12-31","cited_by_count":31,"area":"distill"},{"id":"2208.10160","title":"PANDA: Prompt Transfer Meets Knowledge Distillation for Efficient Model Adaptation","created":"2022-08-22","cited_by_count":31,"area":"distill"},{"id":"2209.10359","title":"Momentum Adversarial Distillation: Handling Large Distribution Shifts in Data-Free Knowledge Distillation","created":"2022-09-21","cited_by_count":31,"area":"distill"},{"id":"2211.04742","title":"Knowledge Distillation for Federated Learning: a Practical Guide","created":"2022-11-09","cited_by_count":31,"area":"distill"},{"id":"2211.13929","title":"XKD: Cross-modal Knowledge Distillation with Domain Alignment for Video Representation Learning","created":"2022-11-25","cited_by_count":31,"area":"distill"},{"id":"2212.10423","title":"Fine-Grained Distillation for Long Document Retrieval","created":"2022-12-20","cited_by_count":29,"area":"distill"},{"id":"2410.17215","title":"MiniPLM: Knowledge Distillation for Pre-Training Language Models","created":"2024-10-22","cited_by_count":29,"area":"distill"},{"id":"2104.06457","title":"Source and Target Bidirectional Knowledge Distillation for End-to-end Speech Translation","created":"2021-04-13","cited_by_count":28,"area":"distill"},{"id":"2203.02721","title":"Consistent Representation Learning for Continual Relation Extraction","created":"2022-03-05","cited_by_count":28,"area":"distill"},{"id":"2105.05912","title":"MATE-KD: Masked Adversarial TExt, a Companion to Knowledge Distillation","created":"2021-05-12","cited_by_count":27,"area":"distill"},{"id":"2210.15274","title":"Improved Feature Distillation via Projector Ensemble","created":"2022-10-27","cited_by_count":27,"area":"distill"},{"id":"2303.17386","title":"Complementary Random Masking for RGB-Thermal Semantic Segmentation","created":"2023-03-30","cited_by_count":27,"area":"distill"},{"id":"2305.08776","title":"Bridging the Domain Gap: Self-Supervised 3D Scene Understanding with Foundation Models","created":"2023-05-15","cited_by_count":27,"area":"distill"},{"id":"2410.16215","title":"Pre-training Distillation for Large Language Models: A Design Space Exploration","created":"2024-10-21","cited_by_count":27,"area":"distill"},{"id":"2108.05507","title":"Distilling Holistic Knowledge with Graph Neural Networks","created":"2021-08-12","cited_by_count":26,"area":"distill"},{"id":"2305.10010","title":"AD-KD: Attribution-Driven Knowledge Distillation for Language Model Compression","created":"2023-05-17","cited_by_count":26,"area":"distill"},{"id":"2308.04268","title":"Teacher-Student Architecture for Knowledge Distillation: A Survey","created":"2023-08-08","cited_by_count":26,"area":"distill"},{"id":"2403.06213","title":"$V_kD:$ Improving Knowledge Distillation using Orthogonal Projections","created":"2024-03-10","cited_by_count":26,"area":"distill"},{"id":"2207.00555","title":"FitHuBERT: Going Thinner and Deeper for Knowledge Distillation of Speech Self-Supervised Learning","created":"2022-07-01","cited_by_count":25,"area":"distill"},{"id":"2302.09632","title":"HomoDistil: Homotopic Task-Agnostic Distillation of Pre-trained Transformers","created":"2023-02-19","cited_by_count":25,"area":"distill"},{"id":"2305.05010","title":"Do Not Blindly Imitate the Teacher: Using Perturbed Loss for Knowledge Distillation","created":"2023-05-08","cited_by_count":25,"area":"distill"},{"id":"2306.04186","title":"Self-supervised Audio Teacher-Student Transformer for Both Clip-level and Frame-level Tasks","created":"2023-06-07","cited_by_count":25,"area":"distill"},{"id":"2402.12842","title":"PromptKD: Distilling Student-Friendly Knowledge for Generative Language Models via Prompt Tuning","created":"2024-02-20","cited_by_count":25,"area":"distill"},{"id":"2406.17633","title":"Knowledge Distillation in Automated Annotation: Supervised Text Classification with LLM-Generated Training Labels","created":"2024-06-25","cited_by_count":25,"area":"distill"},{"id":"2206.08491","title":"Revisiting Self-Distillation","created":"2022-06-17","cited_by_count":24,"area":"distill"},{"id":"2109.05812","title":"UniMS: A Unified Framework for Multimodal Summarization with Knowledge Distillation","created":"2021-09-13","cited_by_count":23,"area":"distill"},{"id":"2109.10504","title":"KD-VLP: Improving End-to-End Vision-and-Language Pretraining with Object Knowledge Distillation","created":"2021-09-22","cited_by_count":23,"area":"distill"},{"id":"2210.17332","title":"Teacher-Student Architecture for Knowledge Learning: A Survey","created":"2022-10-28","cited_by_count":23,"area":"distill"},{"id":"2303.15466","title":"Supervised Masked Knowledge Distillation for Few-Shot Transformers","created":"2023-03-25","cited_by_count":23,"area":"distill"},{"id":"2502.11191","title":"Primus: A Pioneering Collection of Open-Source Datasets for Cybersecurity LLM Training","created":"2025-02-16","cited_by_count":23,"area":"distill"},{"id":"2104.08945","title":"Data-Efficient Language-Supervised Zero-Shot Learning with Self-Distillation","created":"2021-04-18","cited_by_count":22,"area":"distill"},{"id":"2107.02681","title":"VidLanKD: Improving Language Understanding via Video-Distilled Knowledge Transfer","created":"2021-07-06","cited_by_count":22,"area":"distill"},{"id":"2112.03473","title":"Improving Neural Cross-Lingual Summarization via Employing Optimal Transport Distance for Knowledge Distillation","created":"2021-12-07","cited_by_count":22,"area":"distill"},{"id":"2401.04810","title":"Translate-Distill: Learning Cross-Language Dense Retrieval by Translation and Distillation","created":"2024-01-09","cited_by_count":22,"area":"distill"},{"id":"2508.20691","title":"MobileCLIP2: Improving Multi-Modal Reinforced Training","created":"2025-08-28","cited_by_count":21,"area":"distill"},{"id":"2109.06243","title":"KroneckerBERT: Learning Kronecker Decomposition for Pre-trained Language Models via Knowledge Distillation","created":"2021-09-13","cited_by_count":20,"area":"distill"},{"id":"2306.10687","title":"Categories of Response-Based, Feature-Based, and Relation-Based Knowledge Distillation","created":"2023-06-19","cited_by_count":20,"area":"distill"},{"id":"2308.06744","title":"Token-Scaled Logit Distillation for Ternary Weight Generative Language Models","created":"2023-08-13","cited_by_count":20,"area":"distill"},{"id":"2402.12022","title":"Distilling Large Language Models for Text-Attributed Graph Learning","created":"2024-02-19","cited_by_count":20,"area":"distill"},{"id":"2109.10164","title":"RAIL-KD: RAndom Intermediate Layer Mapping for Knowledge Distillation","created":"2021-09-21","cited_by_count":19,"area":"distill"},{"id":"2112.00459","title":"Information Theoretic Representation Distillation","created":"2021-12-01","cited_by_count":19,"area":"distill"},{"id":"2201.10207","title":"SPIRAL: Self-supervised Perturbation-Invariant Representation Learning for Speech Pre-Training","created":"2022-01-25","cited_by_count":19,"area":"distill"},{"id":"2206.01859","title":"Extreme Compression for Pre-trained Transformers Made Simple and Efficient","created":"2022-06-04","cited_by_count":19,"area":"distill"},{"id":"2208.08741","title":"Quantifying the Knowledge in a DNN to Explain Knowledge Distillation for Classification","created":"2022-08-18","cited_by_count":19,"area":"distill"},{"id":"2410.01524","title":"HarmAug: Effective Data Augmentation for Knowledge Distillation of Safety Guard Models","created":"2024-10-02","cited_by_count":19,"area":"distill"},{"id":"2207.08739","title":"Rethinking Data Augmentation for Robust Visual Question Answering","created":"2022-07-18","cited_by_count":18,"area":"distill"},{"id":"2102.04887","title":"NewsBERT: Distilling Pre-trained Language Model for Intelligent News Application","created":"2021-02-09","cited_by_count":17,"area":"distill"},{"id":"2112.00158","title":"Representation learning through cross-modal conditional teacher-student training for speech emotion recognition","created":"2021-11-30","cited_by_count":17,"area":"distill"},{"id":"2202.06212","title":"Uni-Retriever: Towards Learning The Unified Embedding Based Retriever in Bing Sponsored Search","created":"2022-02-13","cited_by_count":17,"area":"distill"},{"id":"2204.00185","title":"Distill-VQ: Learning Retrieval Oriented Vector Quantization By Distilling Knowledge from Dense Embeddings","created":"2022-04-01","cited_by_count":17,"area":"distill"},{"id":"2308.15122","title":"SpikeBERT: A Language Spikformer Learned from BERT with Knowledge Distillation","created":"2023-08-29","cited_by_count":17,"area":"distill"},{"id":"2112.08185","title":"Learning Cross-Lingual IR from an English Retriever","created":"2021-12-15","cited_by_count":16,"area":"distill"},{"id":"2112.14938","title":"Automatic Mixed-Precision Quantization Search of BERT","created":"2021-12-30","cited_by_count":16,"area":"distill"},{"id":"2210.00944","title":"Attention Distillation: self-supervised vision transformer students need more guidance","created":"2022-10-03","cited_by_count":16,"area":"distill"},{"id":"2101.05938","title":"KDLSQ-BERT: A Quantized Bert Combining Knowledge Distillation with Learned Step Size Quantization","created":"2021-01-15","cited_by_count":15,"area":"distill"},{"id":"2106.00241","title":"Reinforced Iterative Knowledge Distillation for Cross-Lingual Named Entity Recognition","created":"2021-06-01","cited_by_count":15,"area":"distill"},{"id":"2109.07222","title":"EfficientBERT: Progressively Searching Multilayer Perceptron via Warm-up Knowledge Distillation","created":"2021-09-15","cited_by_count":15,"area":"distill"},{"id":"2110.03334","title":"Knowledge Distillation for Neural Transducers from Large Self-Supervised Pre-trained Models","created":"2021-10-07","cited_by_count":15,"area":"distill"},{"id":"2110.14131","title":"Temporal Knowledge Distillation for On-device Audio Classification","created":"2021-10-27","cited_by_count":15,"area":"distill"},{"id":"2206.07808","title":"Alexa Teacher Model: Pretraining and Distilling Multi-Billion-Parameter Encoders for Natural Language Understanding Systems","created":"2022-06-15","cited_by_count":15,"area":"distill"},{"id":"2302.04725","title":"Lightweight Transformers for Clinical Natural Language Processing","created":"2023-02-09","cited_by_count":15,"area":"distill"},{"id":"2305.11481","title":"CM-MaskSD: Cross-Modality Masked Self-Distillation for Referring Image Segmentation","created":"2023-05-19","cited_by_count":15,"area":"distill"},{"id":"2306.06629","title":"GKD: A General Knowledge Distillation Framework for Large-scale Pre-trained Language Model","created":"2023-06-11","cited_by_count":15,"area":"distill"},{"id":"2402.17110","title":"Sinkhorn Distance Minimization for Knowledge Distillation","created":"2024-02-27","cited_by_count":15,"area":"distill"}],"next_cursor":"100"}