{"total":63,"rows":[{"id":"2305.15425","title":"Language Model Tokenizers Introduce Unfairness Between Languages","created":"2023-05-17","cited_by_count":214,"area":null},{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2210.03050","title":"State-of-the-art generalisation research in NLP: A taxonomy and review","created":"2022-10-06","cited_by_count":98,"area":null},{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2503.13423","title":"SuperBPE: Space Travel for Language Models","created":"2025-03-17","cited_by_count":63,"area":null},{"id":"2411.14198","title":"Why do language models perform worse for morphologically complex languages?","created":"2024-11-21","cited_by_count":62,"area":null},{"id":"2403.06265","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","created":"2024-03-10","cited_by_count":61,"area":null},{"id":"2312.10523","title":"Paloma: A Benchmark for Evaluating Language Model Fit","created":"2023-12-16","cited_by_count":54,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2405.17067","title":"Tokenization Matters! Degrading Large Language Models through Challenging Their Tokenization","created":"2024-05-27","cited_by_count":32,"area":null},{"id":"2409.04599","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","created":"2024-09-06","cited_by_count":31,"area":null},{"id":"2310.13348","title":"Analyzing Cognitive Plausibility of Subword Tokenization","created":"2023-10-20","cited_by_count":30,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2204.04058","title":"Improving Tokenisation by Alternative Treatment of Spaces","created":"2022-04-08","cited_by_count":19,"area":null},{"id":"2410.12656","title":"Evaluating Morphological Compositional Generalization in Large Language Models","created":"2024-10-16","cited_by_count":19,"area":null},{"id":"2506.19004","title":"Broken Tokens? Your Language Model can Secretly Handle Non-Canonical Tokenizations","created":"2025-06-23","cited_by_count":17,"area":null},{"id":"2410.11627","title":"Tokenization and Morphology in Multilingual Language Models: A Comparative Analysis of mT5 and ByT5","created":"2024-10-15","cited_by_count":14,"area":null},{"id":"2305.05480","title":"Effects of sub-word segmentation on performance of transformer language models","created":"2023-05-09","cited_by_count":12,"area":null},{"id":"2412.13716","title":"Model Decides How to Tokenize: Adaptive DNA Sequence Tokenization with MxDNA","created":"2024-12-18","cited_by_count":12,"area":null},{"id":"2210.07095","title":"Incorporating Context into Subword Vocabularies","created":"2022-10-13","cited_by_count":11,"area":null},{"id":"2402.13432","title":"DrBenchmark: A Large Language Understanding Evaluation Benchmark for French Biomedical Domain","created":"2024-02-20","cited_by_count":11,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2306.17649","title":"Biomedical Language Models are Robust to Sub-optimal Tokenization","created":"2023-06-30","cited_by_count":9,"area":null},{"id":"2310.15113","title":"Counting the Bugs in ChatGPT's Wugs: A Multilingual Investigation into the Morphological Capabilities of a Large Language Model","created":"2023-10-23","cited_by_count":9,"area":null},{"id":"2512.15586","title":"Bolmo: Byteifying the Next Generation of Language Models","created":"2025-12-17","cited_by_count":9,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2207.00982","title":"Linguistically inspired roadmap for building biologically reliable protein language models","created":"2022-07-03","cited_by_count":7,"area":null},{"id":"2305.14214","title":"CompoundPiece: Evaluating and Improving Decompounding Performance of Language Models","created":"2023-05-23","cited_by_count":7,"area":null},{"id":"2404.18543","title":"Time Machine GPT","created":"2024-04-29","cited_by_count":7,"area":null},{"id":"2508.14292","title":"Tokens with Meaning: A Hybrid Tokenization Approach for Turkish","created":"2025-08-19","cited_by_count":7,"area":null},{"id":"2306.15268","title":"Can Pretrained Language Models Derive Correct Semantics from Corrupt Subwords under Noise?","created":"2023-06-27","cited_by_count":6,"area":null},{"id":"2308.13116","title":"Sentence Embedding Models for Ancient Greek Using Multilingual Knowledge Distillation","created":"2023-08-24","cited_by_count":6,"area":"distill"},{"id":"2311.00116","title":"BERTwich: Extending BERT's Capabilities to Model Dialectal and Noisy Text","created":"2023-10-31","cited_by_count":6,"area":null},{"id":"2402.14616","title":"The Impact of Word Splitting on the Semantic Content of Contextualized Word Representations","created":"2024-02-22","cited_by_count":6,"area":null},{"id":"2406.03897","title":"HeSum: a Novel Dataset for Abstractive Text Summarization in Hebrew","created":"2024-06-06","cited_by_count":6,"area":null},{"id":"2311.00658","title":"Explicit Morphological Knowledge Improves Pre-training of Language Models for Hebrew","created":"2023-11-01","cited_by_count":5,"area":null},{"id":"2312.02598","title":"Impact of Tokenization on LLaMa Russian Adaptation","created":"2023-12-05","cited_by_count":5,"area":null},{"id":"2403.13754","title":"Different Tokenization Schemes Lead to Comparable Performance in Spanish Number Agreement","created":"2024-03-20","cited_by_count":5,"area":null},{"id":"2403.17856","title":"Verbing Weirds Language (Models): Evaluation of English Zero-Derivation in Five LLMs","created":"2024-03-26","cited_by_count":5,"area":null},{"id":"2203.08909","title":"Morphological Processing of Low-Resource Languages: Where We Are and What's Next","created":"2022-03-16","cited_by_count":4,"area":null},{"id":"2404.04530","title":"A Morphology-Based Investigation of Positional Encodings","created":"2024-04-06","cited_by_count":4,"area":null},{"id":"2210.12302","title":"What do Large Language Models Learn beyond Language?","created":"2022-10-21","cited_by_count":3,"area":null},{"id":"2202.12832","title":"Morphology Without Borders: Clause-Level Morphology","created":"2022-02-25","cited_by_count":2,"area":null},{"id":"2206.07807","title":"How Adults Understand What Young Children Say","created":"2022-06-15","cited_by_count":2,"area":null},{"id":"2401.07923","title":"Word Boundary Information Isn't Useful for Encoder Language Models","created":"2024-01-15","cited_by_count":2,"area":null},{"id":"2506.11115","title":"Incorporating Domain Knowledge into Materials Tokenization","created":"2025-06-09","cited_by_count":2,"area":null},{"id":"2507.03241","title":"KinyaColBERT: A Lexically Grounded Retrieval Model for Low-Resource Retrieval-Augmented Generation","created":"2025-07-04","cited_by_count":2,"area":"rag"},{"id":"2603.10771","title":"Word Recovery in Large Language Models Enables Character-Level Tokenization Robustness","created":"2026-03-11","cited_by_count":2,"area":null},{"id":"2604.28147","title":"On the Proper Treatment of Units in Surprisal Theory","created":"2026-04-30","cited_by_count":2,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2203.07911","title":"Signal in Noise: Exploring Meaning Encoded in Random Character Sequences with Character-Aware Language Models","created":"2022-03-15","cited_by_count":1,"area":null},{"id":"2408.06675","title":"Latin Treebanks in Review: An Evaluation of Morphological Tagging Across Time","created":"2024-08-13","cited_by_count":1,"area":null},{"id":"2510.06461","title":"Linguistically Informed Tokenization Improves ASR for Underresourced Languages","created":"2025-10-07","cited_by_count":1,"area":null},{"id":"2510.17001","title":"Vocab Diet: Reshaping the Vocabulary of LLMs via Vector Arithmetic","created":"2025-10-19","cited_by_count":1,"area":null},{"id":"2511.09709","title":"Contextual morphologically-guided tokenization for Latin encoder models","created":"2025-11-12","cited_by_count":1,"area":null},{"id":"2604.10665","title":"HeceTokenizer: A Syllable-Based Tokenization Approach for Turkish Retrieval","created":"2026-04-12","cited_by_count":1,"area":null},{"id":"2307.00456","title":"Make Text Unlearnable: Exploiting Effective Patterns to Protect Personal Data","created":"2023-07-02","cited_by_count":0,"area":null},{"id":"2406.15245","title":"Unsupervised Morphological Tree Tokenizer","created":"2024-06-21","cited_by_count":0,"area":null},{"id":"2410.02283","title":"Morphological evaluation of subwords vocabulary used by BETO language model","created":"2024-10-03","cited_by_count":0,"area":null},{"id":"2602.04241","title":"Tokenization and Morphological Fidelity in Uralic NLP: A Cross-Lingual Evaluation","created":"2026-02-04","cited_by_count":0,"area":null},{"id":"2606.15521","title":"Emergent retokenization symmetry in large language models: phenomenology and applications","created":"2026-06-14","cited_by_count":0,"area":null},{"id":"2606.23566","title":"LangMAP: A Language-Adaptive Approach to Tokenization","created":"2026-06-22","cited_by_count":0,"area":null}],"next_cursor":null}