{"total":33,"rows":[{"id":"2411.14198","title":"Why do language models perform worse for morphologically complex languages?","created":"2024-11-21","cited_by_count":62,"area":null},{"id":"2405.07883","title":"Zero-Shot Tokenizer Transfer","created":"2024-05-13","cited_by_count":31,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2411.18553","title":"Retrofitting Large Language Models with Dynamic Tokenization","created":"2024-11-27","cited_by_count":16,"area":null},{"id":"2502.20273","title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","created":"2025-02-27","cited_by_count":15,"area":null},{"id":"2503.19693","title":"AdaptiVocab: Enhancing LLM Efficiency in Focused Domains through Lightweight Vocabulary Adaptation","created":"2025-03-25","cited_by_count":15,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2508.08424","title":"Rethinking Tokenization for Rich Morphology: The Dominance of Unigram over BPE and Morphological Alignment","created":"2025-08-11","cited_by_count":9,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2409.18583","title":"Hit the Sweet Spot! Span-Level Ensemble for Large Language Models","created":"2024-09-27","cited_by_count":8,"area":null},{"id":"2412.06926","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","created":"2024-12-09","cited_by_count":8,"area":null},{"id":"2404.00397","title":"An Analysis of BPE Vocabulary Trimming in Neural Machine Translation","created":"2024-03-30","cited_by_count":7,"area":null},{"id":"2502.12835","title":"Subword models struggle with word learning, but surprisal hides it","created":"2025-02-18","cited_by_count":6,"area":null},{"id":"2503.14433","title":"Splintering Nonconcatenative Languages for Better Tokenization","created":"2025-03-18","cited_by_count":6,"area":null},{"id":"2601.13260","title":"Stop Taking Tokenizers for Granted: They Are Core Design Decisions in Large Language Models","created":"2026-01-19","cited_by_count":6,"area":null},{"id":"2403.13754","title":"Different Tokenization Schemes Lead to Comparable Performance in Spanish Number Agreement","created":"2024-03-20","cited_by_count":5,"area":null},{"id":"2410.01487","title":"Small Language Models Also Work With Small Vocabularies: Probing the Linguistic Abilities of Grapheme- and Phoneme-Based Baby Llamas","created":"2024-10-02","cited_by_count":5,"area":null},{"id":"2511.15709","title":"Tokenisation over Bounded Alphabets is Hard","created":"2025-11-19","cited_by_count":5,"area":null},{"id":"2606.27019","title":"MinGram: A Minimalist Unigram Tokenizer with High Compression and Competitive Morphological Alignment","created":"2026-06-25","cited_by_count":5,"area":null},{"id":"2508.02591","title":"CharBench: Evaluating the Role of Tokenization in Character-Level Tasks","created":"2025-08-04","cited_by_count":3,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2506.18639","title":"ByteSpan: Information-Driven Subword Tokenisation","created":"2025-06-23","cited_by_count":1,"area":null},{"id":"2508.06621","title":"Train It and Forget It: Merge Lists are Unnecessary for BPE Inference in Language Models","created":"2025-08-08","cited_by_count":1,"area":null},{"id":"2601.18536","title":"Evaluating Morphological Plausibility of Subword Tokenization via Statistical Alignment with Morpho-Syntactic Features","created":"2026-01-26","cited_by_count":1,"area":null},{"id":"2607.23362","title":"Joint Optimization for Greedy Longest-match Tokenization","created":"2026-07-25","cited_by_count":1,"area":null},{"id":"2608.00837","title":"Pruned BPE: Post-training Visibility Pruning and Token Reallocation for Byte Pair Encoding","created":"2026-08-01","cited_by_count":1,"area":null},{"id":"2508.02455","title":"TreeRanker: Fast and Model-agnostic Ranking System for Code Suggestions in IDEs","created":"2025-08-04","cited_by_count":0,"area":null},{"id":"2601.11518","title":"How Long Is a Piece of String? A Brief Empirical Analysis of Tokenizers","created":"2026-01-16","cited_by_count":0,"area":null},{"id":"2608.17325","title":"What Tokens are Learned when Tokenization is Optimized Jointly with Language Modeling?","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2609.06898","title":"Dynamic-Programming-Guided Hierarchical BPE and Empirical Analysis of Vocabulary Pruning","created":"2026-09-07","cited_by_count":0,"area":null},{"id":"2609.12960","title":"Fewer Words, Not Fewer Tokens: Measuring the Sanskrit Tokenization Penalty per Proposition","created":"2026-09-11","cited_by_count":0,"area":null},{"id":"2610.01127","title":"Counting and Min-Cost Encoding for Tokenization in Large Language Models","created":"2026-10-01","cited_by_count":0,"area":null}],"next_cursor":null}