{"total":166,"rows":[{"id":"2102.12092","title":"Zero-Shot Text-to-Image Generation","created":"2021-02-24","cited_by_count":4288,"area":null},{"id":"2212.04356","title":"Robust Speech Recognition via Large-Scale Weak Supervision","created":"2022-12-06","cited_by_count":4057,"area":null},{"id":"2207.14255","title":"Efficient Training of Language Models to Fill in the Middle","created":"2022-07-28","cited_by_count":255,"area":null},{"id":"2103.06874","title":"CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation","created":"2021-03-11","cited_by_count":232,"area":null},{"id":"2309.10668","title":"Language Modeling Is Compression","created":"2023-09-19","cited_by_count":225,"area":null},{"id":"2305.13707","title":"Do All Languages Cost the Same? Tokenization in the Era of Commercial Language Models","created":"2023-05-23","cited_by_count":192,"area":null},{"id":"2405.14782","title":"Lessons from the Trenches on Reproducible Evaluation of Language Models","created":"2024-05-23","cited_by_count":179,"area":null},{"id":"2106.12672","title":"Charformer: Fast Character Transformers via Gradient-based Subword Tokenization","created":"2021-06-23","cited_by_count":160,"area":null},{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2109.00486","title":"Survey of Low-Resource Machine Translation","created":"2021-09-01","cited_by_count":92,"area":null},{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2402.01035","title":"Getting the most out of your tokenizer for pre-training and domain adaptation","created":"2024-02-01","cited_by_count":66,"area":null},{"id":"2503.13423","title":"SuperBPE: Space Travel for Language Models","created":"2025-03-17","cited_by_count":63,"area":null},{"id":"2204.08832","title":"Impact of Tokenization on Language Models: An Analysis for Turkish","created":"2022-04-19","cited_by_count":62,"area":null},{"id":"2403.06265","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","created":"2024-03-10","cited_by_count":61,"area":null},{"id":"2306.16842","title":"Tokenization and the Noiseless Channel","created":"2023-06-29","cited_by_count":59,"area":null},{"id":"2207.06991","title":"Language Modelling with Pixels","created":"2022-07-14","cited_by_count":55,"area":null},{"id":"2105.01735","title":"HerBERT: Efficiently Pretrained Transformer-based Language Model for Polish","created":"2021-05-04","cited_by_count":51,"area":null},{"id":"2410.05864","title":"From Tokens to Words: On the Inner Lexicon of LLMs","created":"2024-10-08","cited_by_count":48,"area":null},{"id":"2104.08211","title":"Robust Open-Vocabulary Translation from Visual Text Representations","created":"2021-04-16","cited_by_count":47,"area":null},{"id":"2405.05417","title":"Fishing for Magikarp: Automatically Detecting Under-trained Tokens in Large Language Models","created":"2024-05-08","cited_by_count":46,"area":null},{"id":"2411.03766","title":"Number Cookbook: Number Understanding of Language Models and How to Improve It","created":"2024-11-06","cited_by_count":42,"area":null},{"id":"2106.08226","title":"Consistency Regularization for Cross-Lingual Fine-Tuning","created":"2021-06-15","cited_by_count":41,"area":null},{"id":"2202.05148","title":"Identifying Weaknesses in Machine Translation Metrics Through Minimum Bayes Risk Decoding: A Case Study for COMET","created":"2022-02-10","cited_by_count":41,"area":null},{"id":"2103.08490","title":"Multi-view Subword Regularization","created":"2021-03-15","cited_by_count":40,"area":null},{"id":"2110.00672","title":"Low Frequency Names Exhibit Bias and Overfitting in Contextualizing Language Models","created":"2021-10-01","cited_by_count":39,"area":null},{"id":"2206.02608","title":"What do tokens know about their characters and how do they know it?","created":"2022-06-06","cited_by_count":36,"area":null},{"id":"2305.01778","title":"SLTUNET: A Simple Unified Model for Sign Language Translation","created":"2023-05-02","cited_by_count":35,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2104.00290","title":"Many-to-English Machine Translation Tools, Data, and Pretrained Models","created":"2021-04-01","cited_by_count":31,"area":null},{"id":"2409.04599","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","created":"2024-09-06","cited_by_count":31,"area":null},{"id":"2412.03719","title":"From Language Models over Tokens to Language Models over Characters","created":"2024-12-04","cited_by_count":31,"area":null},{"id":"2109.02550","title":"You should evaluate your language model on marginal likelihood over tokenisations","created":"2021-09-06","cited_by_count":30,"area":null},{"id":"2110.02782","title":"How BPE Affects Memorization in Transformers","created":"2021-10-06","cited_by_count":29,"area":null},{"id":"2110.08191","title":"Why don't people use character-level machine translation?","created":"2021-10-15","cited_by_count":27,"area":null},{"id":"2203.01976","title":"Overlap-based Vocabulary Generation Improves Cross-lingual Transfer Among Related Languages","created":"2022-03-03","cited_by_count":25,"area":null},{"id":"2206.07615","title":"The SIGMORPHON 2022 Shared Task on Morpheme Segmentation","created":"2022-06-15","cited_by_count":25,"area":null},{"id":"2502.00894","title":"MorphBPE: A Morpho-Aware Tokenizer Bridging Linguistic Complexity for Efficient LLM Training Across Morphologies","created":"2025-02-02","cited_by_count":25,"area":null},{"id":"2203.13838","title":"Analyzing Generalization of Vision and Language Navigation to Unseen Outdoor Areas","created":"2022-03-25","cited_by_count":24,"area":null},{"id":"2103.06716","title":"Learning Word-Level Confidence For Subword End-to-End ASR","created":"2021-03-11","cited_by_count":22,"area":null},{"id":"2307.05355","title":"UniCoRN: Unified Cognitive Signal ReconstructioN bridging cognitive signals and human language","created":"2023-07-06","cited_by_count":22,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2504.05747","title":"SEA-LION: Southeast Asian Languages in One Network","created":"2025-04-08","cited_by_count":22,"area":null},{"id":"2408.08541","title":"Where is the signal in tokenization space?","created":"2024-08-16","cited_by_count":21,"area":null},{"id":"2406.11687","title":"Tokenization Falling Short: On Subword Robustness in Large Language Models","created":"2024-06-17","cited_by_count":20,"area":null},{"id":"2109.06772","title":"Improving Zero-shot Cross-lingual Transfer between Closely Related Languages by injecting Character-level Noise","created":"2021-09-14","cited_by_count":19,"area":null},{"id":"2304.10158","title":"Does Manipulating Tokenization Aid Cross-Lingual Transfer? A Study on POS Tagging for Non-Standardized Languages","created":"2023-04-20","cited_by_count":19,"area":null},{"id":"2308.00189","title":"Generative Models as a Complex Systems Science: How can we make sense of large language model behavior?","created":"2023-07-31","cited_by_count":19,"area":null},{"id":"2404.03608","title":"Sailor: Open Language Models for South-East Asia","created":"2024-04-04","cited_by_count":19,"area":null},{"id":"2306.17757","title":"Should you marginalize over possible tokenizations?","created":"2023-06-30","cited_by_count":18,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2502.13581","title":"ActionPiece: Contextually Tokenizing Action Sequences for Generative Recommendation","created":"2025-02-19","cited_by_count":18,"area":null},{"id":"2206.03352","title":"Searching for Optimal Subword Tokenization in Cross-domain NER","created":"2022-06-07","cited_by_count":17,"area":null},{"id":"2506.19004","title":"Broken Tokens? Your Language Model can Secretly Handle Non-Canonical Tokenizations","created":"2025-06-23","cited_by_count":17,"area":null},{"id":"2206.02457","title":"Improving Contrastive Learning of Sentence Embeddings with Case-Augmented Positives and Retrieved Negatives","created":"2022-06-06","cited_by_count":16,"area":null},{"id":"2404.17808","title":"Scaffold-BPE: Enhancing Byte Pair Encoding for Large Language Models with Simple and Effective Scaffold Token Removal","created":"2024-04-27","cited_by_count":16,"area":null},{"id":"2105.12410","title":"Joint Optimization of Tokenization and Downstream Model","created":"2021-05-26","cited_by_count":15,"area":null},{"id":"2111.11133","title":"L-Verse: Bidirectional Generation Between Image and Text","created":"2021-11-22","cited_by_count":15,"area":null},{"id":"2410.09303","title":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","created":"2024-10-11","cited_by_count":15,"area":null},{"id":"2106.06125","title":"Bridging Subword Gaps in Pretrain-Finetune Paradigm for Natural Language Generation","created":"2021-06-11","cited_by_count":14,"area":null},{"id":"2112.14569","title":"Fine-Tuning Transformers: Vocabulary Transfer","created":"2021-12-29","cited_by_count":14,"area":null},{"id":"2402.14614","title":"Two Counterexamples to Tokenization and the Noiseless Channel","created":"2024-02-22","cited_by_count":13,"area":null},{"id":"2109.04556","title":"Subword Mapping and Anchoring across Languages","created":"2021-09-09","cited_by_count":12,"area":null},{"id":"2410.01188","title":"Gold Panning in Vocabulary: An Adaptive Method for Vocabulary Expansion of Domain-Specific LLMs","created":"2024-10-02","cited_by_count":12,"area":null},{"id":"2210.07095","title":"Incorporating Context into Subword Vocabularies","created":"2022-10-13","cited_by_count":11,"area":null},{"id":"2310.03477","title":"Tik-to-Tok: Translating Language Models One Token at a Time: An Embedding Initialization Strategy for Efficient Language Adaptation","created":"2023-10-05","cited_by_count":11,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2204.00665","title":"CipherDAug: Ciphertext based Data Augmentation for Neural Machine Translation","created":"2022-04-01","cited_by_count":9,"area":null},{"id":"2308.00683","title":"CodeBPE: Investigating Subtokenization Options for Large Language Model Pretraining on Source Code","created":"2023-08-01","cited_by_count":9,"area":null},{"id":"2109.05772","title":"Wine is Not v i n. -- On the Compatibility of Tokenizations Across Languages","created":"2021-09-13","cited_by_count":8,"area":null},{"id":"2111.08634","title":"NVIDIA NeMo Neural Machine Translation Systems for English-German and English-Russian News and Biomedical Tasks at WMT21","created":"2021-11-16","cited_by_count":8,"area":null},{"id":"2209.04126","title":"MaxMatch-Dropout: Subword Regularization for WordPiece","created":"2022-09-09","cited_by_count":8,"area":null},{"id":"2210.08779","title":"Towards Summary Candidates Fusion","created":"2022-10-17","cited_by_count":8,"area":null},{"id":"2305.18294","title":"Transformer Language Models Handle Word Frequency in Prediction Head","created":"2023-05-29","cited_by_count":8,"area":null},{"id":"2306.01497","title":"Data-Efficient French Language Modeling with CamemBERTa","created":"2023-06-02","cited_by_count":8,"area":null},{"id":"2310.11628","title":"Learn Your Tokens: Word-Pooled Tokenization for Language Modeling","created":"2023-10-17","cited_by_count":8,"area":null},{"id":"2109.01100","title":"How Suitable Are Subword Segmentation Strategies for Translating Non-Concatenative Morphology?","created":"2021-09-02","cited_by_count":7,"area":null},{"id":"2212.09912","title":"Tokenization Consistency Matters for Generative Models on Extractive NLP Tasks","created":"2022-12-19","cited_by_count":7,"area":null},{"id":"2305.17906","title":"Byte-Level Grammatical Error Correction Using Synthetic and Curated Corpora","created":"2023-05-29","cited_by_count":7,"area":null},{"id":"2404.00397","title":"An Analysis of BPE Vocabulary Trimming in Neural Machine Translation","created":"2024-03-30","cited_by_count":7,"area":null},{"id":"2406.07096","title":"Fast Context-Biasing for CTC and Transducer ASR models with CTC-based Word Spotter","created":"2024-06-11","cited_by_count":7,"area":null},{"id":"2407.16382","title":"TookaBERT: A Step Forward for Persian NLU","created":"2024-07-23","cited_by_count":7,"area":null},{"id":"2506.01687","title":"StochasTok: Improving Fine-Grained Subword Understanding in LLMs","created":"2025-06-02","cited_by_count":7,"area":null},{"id":"2506.14123","title":"Sampling from Your Language Model One Byte at a Time","created":"2025-06-17","cited_by_count":7,"area":null},{"id":"2508.15390","title":"Exploiting Vocabulary Frequency Imbalance in Language Model Pre-training","created":"2025-08-21","cited_by_count":7,"area":null},{"id":"2211.12781","title":"Breaking the Representation Bottleneck of Chinese Characters: Neural Machine Translation with Stroke Sequence Modeling","created":"2022-11-23","cited_by_count":6,"area":null},{"id":"2311.08966","title":"Improving Large-scale Deep Biasing with Phoneme Features and Text-only Data in Streaming Transducer","created":"2023-11-15","cited_by_count":6,"area":null},{"id":"2205.11490","title":"Local Byte Fusion for Neural Machine Translation","created":"2022-05-23","cited_by_count":5,"area":null},{"id":"2210.05610","title":"MTet: Multi-domain Translation for English and Vietnamese","created":"2022-10-11","cited_by_count":5,"area":null},{"id":"2306.01393","title":"Assessing the Importance of Frequency versus Compositionality for Subword-based Tokenization in NMT","created":"2023-06-02","cited_by_count":5,"area":null},{"id":"2404.06508","title":"On the Effect of (Near) Duplicate Subwords in Language Modelling","created":"2024-04-09","cited_by_count":5,"area":null},{"id":"2404.08259","title":"Investigating Neural Machine Translation for Low-Resource Languages: Using Bavarian as a Case Study","created":"2024-04-12","cited_by_count":5,"area":null},{"id":"2501.06246","title":"A Partition Cover Approach to Tokenization","created":"2025-01-08","cited_by_count":5,"area":null},{"id":"2104.00041","title":"Domain-specific MT for Low-resource Languages: The case of Bambara-French","created":"2021-03-31","cited_by_count":4,"area":null},{"id":"2306.07764","title":"Tokenization with Factorized Subword Encoding","created":"2023-06-13","cited_by_count":4,"area":null},{"id":"2401.15360","title":"Importance-Aware Data Augmentation for Document-Level Neural Machine Translation","created":"2024-01-27","cited_by_count":4,"area":null},{"id":"2406.09717","title":"UniBridge: A Unified Approach to Cross-Lingual Transfer Learning for Low-Resource Languages","created":"2024-06-14","cited_by_count":4,"area":null},{"id":"2408.11443","title":"Distributional Properties of Subword Regularization","created":"2024-08-21","cited_by_count":4,"area":null},{"id":"2512.12641","title":"Which Pieces Does Unigram Tokenization Really Need?","created":"2025-12-14","cited_by_count":4,"area":null},{"id":"2512.20757","title":"TokSuite: Measuring the Impact of Tokenizer Choice on Language Model Behavior","created":"2025-12-23","cited_by_count":4,"area":null}],"next_cursor":"100"}