{"total":107,"rows":[{"id":"2412.13663","title":"Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","created":"2024-12-18","cited_by_count":554,"area":null},{"id":"2205.05131","title":"UL2: Unifying Language Learning Paradigms","created":"2022-05-10","cited_by_count":295,"area":null},{"id":"2404.07965","title":"Rho-1: Not All Tokens Are What You Need","created":"2024-04-11","cited_by_count":133,"area":null},{"id":"2302.00861","title":"SimMTM: A Simple Pre-Training Framework for Masked Time-Series Modeling","created":"2023-02-02","cited_by_count":115,"area":null},{"id":"2205.10287","title":"On the SDEs and Scaling Rules for Adaptive Gradient Algorithms","created":"2022-05-20","cited_by_count":104,"area":null},{"id":"2303.11504","title":"Language Model Behavior: A Comprehensive Survey","created":"2023-03-20","cited_by_count":92,"area":null},{"id":"2504.08165","title":"Findings of the BabyLM Challenge: Sample-Efficient Pretraining on Developmentally Plausible Corpora","created":"2025-04-10","cited_by_count":88,"area":null},{"id":"2212.09803","title":"Training Trajectories of Language Models Across Scales","created":"2022-12-19","cited_by_count":81,"area":null},{"id":"2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","created":"2022-12-28","cited_by_count":81,"area":null},{"id":"2209.00099","title":"Efficient Methods for Natural Language Processing: A Survey","created":"2022-08-31","cited_by_count":78,"area":null},{"id":"2209.02128","title":"Evaluating the Susceptibility of Pre-Trained Language Models via Handcrafted Adversarial Examples","created":"2022-09-05","cited_by_count":69,"area":null},{"id":"2208.02131","title":"Masked Vision and Language Modeling for Multi-modal Representation Learning","created":"2022-08-03","cited_by_count":67,"area":null},{"id":"2306.02069","title":"MultiLegalPile: A 689GB Multilingual Legal Corpus","created":"2023-06-03","cited_by_count":60,"area":null},{"id":"2208.00173","title":"A Survey on Masked Autoencoder for Self-supervised Learning in Vision and Beyond","created":"2022-07-30","cited_by_count":58,"area":null},{"id":"2306.11489","title":"Give Us the Facts: Enhancing Large Language Models with Knowledge Graphs for Fact-aware Language Modeling","created":"2023-06-20","cited_by_count":56,"area":null},{"id":"2207.06991","title":"Language Modelling with Pixels","created":"2022-07-14","cited_by_count":55,"area":null},{"id":"2305.07507","title":"LeXFiles and LegalLAMA: Facilitating English Multinational Legal Language Model Development","created":"2023-05-12","cited_by_count":54,"area":null},{"id":"2212.10544","title":"Pretraining Without Attention","created":"2022-12-20","cited_by_count":51,"area":null},{"id":"2303.08117","title":"Do Transformers Parse while Predicting the Masked Word?","created":"2023-03-14","cited_by_count":48,"area":null},{"id":"2211.09120","title":"AdaMAE: Adaptive Masking for Efficient Spatiotemporal Learning with Masked Autoencoders","created":"2022-11-16","cited_by_count":43,"area":null},{"id":"2305.06212","title":"Privacy-Preserving Parameter-Efficient Fine-Tuning for Large Language Model Services","created":"2023-05-10","cited_by_count":43,"area":null},{"id":"2312.17482","title":"MosaicBERT: A Bidirectional Encoder Optimized for Fast Pretraining","created":"2023-12-29","cited_by_count":34,"area":null},{"id":"2204.02964","title":"Unleashing Vanilla Vision Transformer with Masked Image Modeling for Object Detection","created":"2022-04-06","cited_by_count":33,"area":null},{"id":"2503.05500","title":"EuroBERT: Scaling Multilingual Encoders for European Languages","created":"2025-03-07","cited_by_count":31,"area":null},{"id":"2306.03009","title":"Using Sequences of Life-events to Predict Human Lives","created":"2023-06-05","cited_by_count":27,"area":null},{"id":"2210.06432","title":"InfoCSE: Information-aggregated Contrastive Learning of Sentence Embeddings","created":"2022-10-08","cited_by_count":26,"area":null},{"id":"2211.09799","title":"CAE v2: Context Autoencoder with CLIP Target","created":"2022-11-17","cited_by_count":25,"area":null},{"id":"2406.05183","title":"The Factorization Curse: Which Tokens You Predict Underlie the Reversal Curse and More","created":"2024-06-07","cited_by_count":25,"area":null},{"id":"2312.01017","title":"Unveiling the Power of Audio-Visual Early Fusion Transformers with Dense Interactions through Masked Modeling","created":"2023-12-02","cited_by_count":24,"area":null},{"id":"2306.00788","title":"Understanding Augmentation-based Self-Supervised Representation Learning via RKHS Approximation and Regression","created":"2023-06-01","cited_by_count":20,"area":null},{"id":"2210.11298","title":"Tele-Knowledge Pre-training for Fault Analysis","created":"2022-10-20","cited_by_count":18,"area":null},{"id":"2307.09270","title":"Linearized Relative Positional Encoding","created":"2023-07-18","cited_by_count":17,"area":null},{"id":"2208.07670","title":"ConTextual Masked Auto-Encoder for Dense Passage Retrieval","created":"2022-08-16","cited_by_count":16,"area":null},{"id":"2507.00994","title":"Should We Still Pretrain Encoders with Masked Language Modeling?","created":"2025-07-01","cited_by_count":16,"area":null},{"id":"2208.10806","title":"Learning Better Masking for Better Language Model Pre-training","created":"2022-08-23","cited_by_count":15,"area":null},{"id":"2210.11771","title":"InforMask: Unsupervised Informative Masking for Language Model Pretraining","created":"2022-10-21","cited_by_count":15,"area":null},{"id":"2310.12580","title":"Pretraining Language Models with Text-Attributed Heterogeneous Graphs","created":"2023-10-19","cited_by_count":15,"area":null},{"id":"2402.14073","title":"Improving Language Understanding from Screenshots","created":"2024-02-21","cited_by_count":14,"area":null},{"id":"2410.13516","title":"PORTAL: Scalable Tabular Foundation Models via Content-Specific Tokenization","created":"2024-10-17","cited_by_count":14,"area":null},{"id":"2305.15096","title":"Dynamic Masking Rate Schedules for MLM Pretraining","created":"2023-05-24","cited_by_count":12,"area":null},{"id":"2504.07373","title":"ChronoFormer: Time-Aware Transformer Architectures for Structured Clinical Event Modeling","created":"2025-04-10","cited_by_count":12,"area":null},{"id":"2310.11166","title":"ViSoBERT: A Pre-Trained Language Model for Vietnamese Social Media Text Processing","created":"2023-10-17","cited_by_count":11,"area":null},{"id":"2208.04060","title":"GRIT-VLP: Grouped Mini-batch Sampling for Efficient Vision and Language Pre-training","created":"2022-08-08","cited_by_count":10,"area":null},{"id":"2211.13252","title":"Mask the Correct Tokens: An Embarrassingly Simple Approach for Error Correction","created":"2022-11-23","cited_by_count":10,"area":null},{"id":"2302.02060","title":"Representation Deficiency in Masked Language Modeling","created":"2023-02-04","cited_by_count":10,"area":null},{"id":"2309.10738","title":"MelodyGLM: Multi-task Pre-training for Symbolic Melody Generation","created":"2023-09-19","cited_by_count":10,"area":null},{"id":"2404.08330","title":"Emerging Property of Masked Token for Effective Pre-training","created":"2024-04-12","cited_by_count":10,"area":null},{"id":"2212.09598","title":"Query-as-context Pre-training for Dense Passage Retrieval","created":"2022-12-19","cited_by_count":9,"area":null},{"id":"2210.12540","title":"EntityCS: Improving Zero-Shot Cross-lingual Transfer with Entity-Centric Code Switching","created":"2022-10-22","cited_by_count":8,"area":null},{"id":"2307.10156","title":"Exploring Transformer Extrapolation","created":"2023-07-19","cited_by_count":8,"area":null},{"id":"2410.03884","title":"KidLM: Advancing Language Models for Children -- Early Insights and Future Directions","created":"2024-10-04","cited_by_count":8,"area":null},{"id":"2211.04898","title":"Mask More and Mask Later: Efficient Pre-training of Masked Language Models by Disentangling the [MASK] Token","created":"2022-11-09","cited_by_count":7,"area":null},{"id":"2303.10494","title":"Revisiting the Plastic Surgery Hypothesis via Large Language Models","created":"2023-03-18","cited_by_count":7,"area":null},{"id":"2311.01955","title":"Too Much Information: Keeping Training Simple for BabyLMs","created":"2023-11-03","cited_by_count":7,"area":null},{"id":"2212.10729","title":"UnICLAM:Contrastive Representation Learning with Adversarial Masking for Unified and Interpretable Medical Vision Question Answering","created":"2022-12-21","cited_by_count":6,"area":null},{"id":"2301.00230","title":"Disjoint Masking with Joint Distillation for Efficient Masked Image Modeling","created":"2022-12-31","cited_by_count":6,"area":null},{"id":"2311.08756","title":"Accelerating Toeplitz Neural Network with Constant-time Inference Complexity","created":"2023-11-15","cited_by_count":6,"area":null},{"id":"2210.11807","title":"Is Encoder-Decoder Redundant for Neural Machine Translation?","created":"2022-10-21","cited_by_count":5,"area":null},{"id":"2306.00520","title":"On Masked Pre-training and the Marginal Likelihood","created":"2023-06-01","cited_by_count":5,"area":null},{"id":"2308.04118","title":"Multimodal Color Recommendation in Vector Graphic Documents","created":"2023-08-08","cited_by_count":5,"area":null},{"id":"2401.17514","title":"How Useful is Continued Pre-Training for Generative Unsupervised Domain Adaptation?","created":"2024-01-31","cited_by_count":5,"area":null},{"id":"2407.01091","title":"M2QA: Multi-domain Multilingual Question Answering","created":"2024-07-01","cited_by_count":5,"area":null},{"id":"2211.04427","title":"Word Order Matters when you Increase Masking","created":"2022-11-08","cited_by_count":4,"area":null},{"id":"2312.02803","title":"Leveraging Domain Adaptation and Data Augmentation to Improve Qur'anic IR in English and Arabic","created":"2023-12-05","cited_by_count":4,"area":null},{"id":"2310.04632","title":"Automatic Anonymization of Swiss Federal Supreme Court Rulings","created":"2023-10-07","cited_by_count":3,"area":null},{"id":"2409.07889","title":"BLens: Contrastive Captioning of Binary Functions using Ensemble Embedding","created":"2024-09-12","cited_by_count":3,"area":null},{"id":"2502.01184","title":"FragmentNet: Adaptive Graph Fragmentation for Graph-to-Sequence Molecular Representation Learning","created":"2025-02-03","cited_by_count":3,"area":null},{"id":"2502.12953","title":"Task-Informed Anti-Curriculum by Masking Improves Downstream Performance on Text","created":"2025-02-18","cited_by_count":3,"area":null},{"id":"2509.21650","title":"Understanding and Enhancing Mask-Based Pretraining towards Universal Representations","created":"2025-09-25","cited_by_count":3,"area":null},{"id":"2212.05764","title":"Domain Adaptation of Transformer-Based Models using Unlabeled Data for Relevance and Polarity Classification of German Customer Feedback","created":"2022-12-12","cited_by_count":2,"area":null},{"id":"2305.07565","title":"A Memory Model for Question Answering from Streaming Data Supported by Rehearsal and Anticipation of Coreference Information","created":"2023-05-12","cited_by_count":2,"area":null},{"id":"2308.08625","title":"BIOptimus: Pre-training an Optimal Biomedical Language Model with Curriculum Learning for Named Entity Recognition","created":"2023-08-16","cited_by_count":2,"area":null},{"id":"2402.17682","title":"NextLevelBERT: Masked Language Modeling with Higher-Level Representations for Long Documents","created":"2024-02-27","cited_by_count":2,"area":null},{"id":"2408.07180","title":"Unlocking Efficiency: Adaptive Masking for Gene Transformer Models","created":"2024-08-13","cited_by_count":2,"area":null},{"id":"2509.14926","title":"Patent Language Model Pretraining with ModernBERT","created":"2025-09-18","cited_by_count":2,"area":null},{"id":"2307.07258","title":"Improving BERT with Hybrid Pooling Network and Drop Mask","created":"2023-07-14","cited_by_count":1,"area":null},{"id":"2406.11249","title":"Relational Learning in Pre-Trained Models: A Theory from Hypergraph Recovery Perspective","created":"2024-06-17","cited_by_count":1,"area":null},{"id":"2407.12825","title":"A Depression Detection Method Based on Multi-Modal Feature Fusion Using Cross-Attention","created":"2024-07-02","cited_by_count":1,"area":null},{"id":"2410.11071","title":"An Annotated Dataset of Errors in Premodern Greek and Baselines for Detecting Them","created":"2024-10-14","cited_by_count":1,"area":null},{"id":"2412.11477","title":"NoteContrast: Contrastive Language-Diagnostic Pretraining for Medical Text","created":"2024-12-16","cited_by_count":1,"area":null},{"id":"2501.05700","title":"Linguistic Entity Masking to Improve Cross-Lingual Representation of Multilingual Language Models for Low-Resource Languages","created":"2025-01-10","cited_by_count":1,"area":null},{"id":"2504.15160","title":"The Synthetic Imputation Approach: Generating Optimal Synthetic Texts For Underrepresented Categories In Supervised Classification Tasks","created":"2025-04-21","cited_by_count":1,"area":null},{"id":"2505.04643","title":"Prediction-powered estimators for finite population statistics in highly imbalanced textual data: Public hate crime estimation","created":"2025-05-05","cited_by_count":1,"area":null},{"id":"2512.19528","title":"Multi-Modal Soccer Scene Analysis with Masked Pre-Training","created":"2025-12-22","cited_by_count":1,"area":null},{"id":"2601.19926","title":"The Grammar of Transformers: A Systematic Review of Interpretability Research on Syntactic Knowledge in Language Models","created":"2026-01-09","cited_by_count":1,"area":null},{"id":"2605.19568","title":"m3BERT: A Modern, Multi-lingual, Matryoshka Bidirectional Encoder","created":"2026-05-19","cited_by_count":1,"area":null},{"id":"2607.04011","title":"Separating Representation from Reconstruction Enables Scalable Text Encoders","created":"2026-07-04","cited_by_count":1,"area":null},{"id":"2208.08386","title":"Neural Embeddings for Text","created":"2022-08-17","cited_by_count":0,"area":null},{"id":"2303.15619","title":"Typhoon: Towards an Effective Task-Specific Masking Strategy for Pre-trained Language Models","created":"2023-03-27","cited_by_count":0,"area":null},{"id":"2305.18725","title":"AdapterEM: Pre-trained Language Model Adaptation for Generalized Entity Matching using Adapter-tuning","created":"2023-05-30","cited_by_count":0,"area":null},{"id":"2310.14633","title":"Extending Input Contexts of Language Models through Training on Segmented Sequences","created":"2023-10-23","cited_by_count":0,"area":null},{"id":"2403.18025","title":"Improving Pre-trained Language Model Sensitivity via Mask Specific losses: A case study on Biomedical NER","created":"2024-03-26","cited_by_count":0,"area":null},{"id":"2404.17877","title":"PromptCL: Improving Event Representation via Prompt Template and Contrastive Learning","created":"2024-04-27","cited_by_count":0,"area":null},{"id":"2405.12711","title":"A Masked Semi-Supervised Learning Approach for Otago Micro Labels Recognition","created":"2024-05-21","cited_by_count":0,"area":null},{"id":"2406.04156","title":"Pointer-Guided Pre-Training: Infusing Large Language Models with Paragraph-Level Contextual Awareness","created":"2024-06-06","cited_by_count":0,"area":null},{"id":"2410.15592","title":"CPE-Pro: A Structure-Sensitive Deep Learning Method for Protein Representation and Origin Evaluation","created":"2024-10-21","cited_by_count":0,"area":null},{"id":"2410.24201","title":"LingGen: Scalable Multi-Attribute Linguistic Control via Power-Law Masking","created":"2024-10-31","cited_by_count":0,"area":null},{"id":"2411.16618","title":"StructFormer: Document Structure-based Masked Attention and its Impact on Language Model Pre-Training","created":"2024-11-25","cited_by_count":0,"area":null},{"id":"2505.14523","title":"Exploring Graph Representations of Logical Forms for Language Modeling","created":"2025-05-20","cited_by_count":0,"area":null},{"id":"2506.23524","title":"NEU-ESC: A Comprehensive Vietnamese dataset for Educational Sentiment analysis and topic Classification toward multitask learning","created":"2025-06-30","cited_by_count":0,"area":null}],"next_cursor":"100"}