{"total":27,"rows":[{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2210.03050","title":"State-of-the-art generalisation research in NLP: A taxonomy and review","created":"2022-10-06","cited_by_count":98,"area":null},{"id":"2206.02608","title":"What do tokens know about their characters and how do they know it?","created":"2022-06-06","cited_by_count":36,"area":null},{"id":"2405.07883","title":"Zero-Shot Tokenizer Transfer","created":"2024-05-13","cited_by_count":31,"area":null},{"id":"2409.15452","title":"CUTE: Measuring LLMs' Understanding of Their Tokens","created":"2024-09-23","cited_by_count":31,"area":null},{"id":"2212.10474","title":"ByGPT5: End-to-End Style-conditioned Poetry Generation with Token-free Language Models","created":"2022-12-20","cited_by_count":25,"area":null},{"id":"2410.12656","title":"Evaluating Morphological Compositional Generalization in Large Language Models","created":"2024-10-16","cited_by_count":19,"area":null},{"id":"2212.09897","title":"Inducing Character-level Structure in Subword-based Language Models with Type-level Interchange Intervention Training","created":"2022-12-19","cited_by_count":16,"area":null},{"id":"2302.14220","title":"Are Character-level Translations Worth the Wait? Comparing ByT5 and mT5 for Machine Translation","created":"2023-02-28","cited_by_count":15,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2305.05480","title":"Effects of sub-word segmentation on performance of transformer language models","created":"2023-05-09","cited_by_count":12,"area":null},{"id":"2406.16508","title":"Large Vocabulary Size Improves Large Language Models","created":"2024-06-24","cited_by_count":11,"area":null},{"id":"2212.01304","title":"Subword-Delimited Downsampling for Better Character-Level Translation","created":"2022-12-02","cited_by_count":7,"area":null},{"id":"2305.07005","title":"Subword Segmental Machine Translation: Unifying Segmentation and Target Sentence Generation","created":"2023-05-11","cited_by_count":7,"area":null},{"id":"2305.17906","title":"Byte-Level Grammatical Error Correction Using Synthetic and Curated Corpora","created":"2023-05-29","cited_by_count":7,"area":null},{"id":"2309.12491","title":"Exploring the Impact of Training Data Distribution and Subword Tokenization on Gender Bias in Machine Translation","created":"2023-09-21","cited_by_count":7,"area":null},{"id":"2504.02122","title":"Overcoming Vocabulary Constraints with Pixel-level Fallback","created":"2025-04-02","cited_by_count":6,"area":null},{"id":"2205.11490","title":"Local Byte Fusion for Neural Machine Translation","created":"2022-05-23","cited_by_count":5,"area":null},{"id":"2306.01393","title":"Assessing the Importance of Frequency versus Compositionality for Subword-based Tokenization in NMT","created":"2023-06-02","cited_by_count":5,"area":null},{"id":"2511.05578","title":"UTF-8 Plumbing: Byte-level Tokenizers Unavoidably Enable LLMs to Generate Ill-formed UTF-8","created":"2025-11-05","cited_by_count":3,"area":null},{"id":"2505.24561","title":"Improving Language and Modality Transfer in Translation by Character-level Modeling","created":"2025-05-30","cited_by_count":2,"area":null},{"id":"2601.11643","title":"Syllabic Agglutinative Tokenizations for Indonesian LLM: A Study from Gasing Literacy Learning System","created":"2026-01-14","cited_by_count":2,"area":null},{"id":"2305.05214","title":"CharSpan: Utilizing Lexical Similarity to Enable Zero-Shot Machine Translation for Extremely Low-resource Languages","created":"2023-05-09","cited_by_count":1,"area":null},{"id":"2403.17220","title":"Making Sentence Embeddings Robust to User-Generated Content","created":"2024-03-25","cited_by_count":1,"area":null},{"id":"2308.04398","title":"Character-level NMT and language similarity","created":"2023-08-08","cited_by_count":0,"area":null},{"id":"2604.16689","title":"The Query Channel: Information-Theoretic Limits of Masking-Based Explanations","created":"2026-04-17","cited_by_count":0,"area":null},{"id":"2609.00463","title":"Toppling the Hierarchy in Byte-level Language Modeling","created":"2026-08-31","cited_by_count":0,"area":null}],"next_cursor":null}