{"total":29,"rows":[{"id":"2403.06265","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","created":"2024-03-10","cited_by_count":61,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2409.04599","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","created":"2024-09-06","cited_by_count":31,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2502.20273","title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","created":"2025-02-27","cited_by_count":15,"area":null},{"id":"2503.19693","title":"AdaptiVocab: Enhancing LLM Efficiency in Focused Domains through Lightweight Vocabulary Adaptation","created":"2025-03-25","cited_by_count":15,"area":null},{"id":"2410.02691","title":"On the Proper Treatment of Tokenization in Psycholinguistics","created":"2024-10-03","cited_by_count":13,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2508.08424","title":"Rethinking Tokenization for Rich Morphology: The Dominance of Unigram over BPE and Morphological Alignment","created":"2025-08-11","cited_by_count":9,"area":null},{"id":"2512.15586","title":"Bolmo: Byteifying the Next Generation of Language Models","created":"2025-12-17","cited_by_count":9,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2404.00397","title":"An Analysis of BPE Vocabulary Trimming in Neural Machine Translation","created":"2024-03-30","cited_by_count":7,"area":null},{"id":"2502.12835","title":"Subword models struggle with word learning, but surprisal hides it","created":"2025-02-18","cited_by_count":6,"area":null},{"id":"2503.14433","title":"Splintering Nonconcatenative Languages for Better Tokenization","created":"2025-03-18","cited_by_count":6,"area":null},{"id":"2504.03036","title":"IPA-CHILDES & G2P+: Feature-Rich Resources for Cross-Lingual Phonology and Phonemic Language Modeling","created":"2025-04-03","cited_by_count":6,"area":null},{"id":"2601.13260","title":"Stop Taking Tokenizers for Granted: They Are Core Design Decisions in Large Language Models","created":"2026-01-19","cited_by_count":6,"area":null},{"id":"2402.09808","title":"Knowledge of Pretrained Language Models on Surface Information of Tokens","created":"2024-02-15","cited_by_count":5,"area":null},{"id":"2410.01487","title":"Small Language Models Also Work With Small Vocabularies: Probing the Linguistic Abilities of Grapheme- and Phoneme-Based Baby Llamas","created":"2024-10-02","cited_by_count":5,"area":null},{"id":"2603.05193","title":"Transducing Language Models","created":"2026-03-05","cited_by_count":5,"area":null},{"id":"2510.09947","title":"Beyond Fertility: Analyzing STRR as a Metric for Multilingual Tokenization Evaluation","created":"2025-10-11","cited_by_count":3,"area":null},{"id":"2504.03338","title":"BabyLM's First Words: Word Segmentation as a Phonological Probing Task","created":"2025-04-04","cited_by_count":2,"area":null},{"id":"2601.04469","title":"SampoNLP: A Self-Referential Toolkit for Morphological Analysis of Subword Tokenizers","created":"2026-01-08","cited_by_count":2,"area":null},{"id":"2604.28147","title":"On the Proper Treatment of Units in Surprisal Theory","created":"2026-04-30","cited_by_count":2,"area":null},{"id":"2606.20993","title":"Phonemes to the Rescue: Multilingual Tokenization Based on International Phonetic Alphabet","created":"2026-06-18","cited_by_count":2,"area":null},{"id":"2506.18639","title":"ByteSpan: Information-Driven Subword Tokenisation","created":"2025-06-23","cited_by_count":1,"area":null},{"id":"2508.06621","title":"Train It and Forget It: Merge Lists are Unnecessary for BPE Inference in Language Models","created":"2025-08-08","cited_by_count":1,"area":null},{"id":"2505.16277","title":"Spontaneous Speech Variables for Evaluating LLMs Cognitive Plausibility","created":"2025-05-22","cited_by_count":0,"area":null},{"id":"2603.19427","title":"Vocabulary shapes cross-lingual variation of word-order learnability in language models","created":"2026-03-19","cited_by_count":0,"area":null},{"id":"2607.15209","title":"Expanding the Lexicon of Ge'ez Based African Languages: A Comparative Study of Amharic and Tigrinya","created":"2026-07-16","cited_by_count":0,"area":null}],"next_cursor":null}