{"total":59,"rows":[{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2402.19449","title":"Heavy-Tailed Class Imbalance and Why Adam Outperforms Gradient Descent on Language Models","created":"2024-02-29","cited_by_count":80,"area":null},{"id":"2410.05864","title":"From Tokens to Words: On the Inner Lexicon of LLMs","created":"2024-10-08","cited_by_count":48,"area":null},{"id":"2406.14561","title":"How to Compute the Probability of a Word","created":"2024-06-20","cited_by_count":36,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2409.04599","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","created":"2024-09-06","cited_by_count":31,"area":null},{"id":"2310.05317","title":"Task-Adaptive Tokenization: Enhancing Long-Form Text Generation Efficacy in Mental Health and Beyond","created":"2023-10-09","cited_by_count":23,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2402.14327","title":"Subobject-level Image Tokenization","created":"2024-02-22","cited_by_count":21,"area":null},{"id":"2502.18795","title":"Anything Goes? A Crosslinguistic Study of (Im)possible Language Learning in LMs","created":"2025-02-26","cited_by_count":20,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2506.03149","title":"Causal Estimation of Tokenisation Bias","created":"2025-06-03","cited_by_count":16,"area":null},{"id":"2410.09303","title":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","created":"2024-10-11","cited_by_count":15,"area":null},{"id":"2502.20273","title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","created":"2025-02-27","cited_by_count":15,"area":null},{"id":"2503.19693","title":"AdaptiVocab: Enhancing LLM Efficiency in Focused Domains through Lightweight Vocabulary Adaptation","created":"2025-03-25","cited_by_count":15,"area":null},{"id":"2511.01380","title":"Confounding Factors in Relating Model Performance to Morphology","created":"2025-11-03","cited_by_count":14,"area":null},{"id":"2406.13560","title":"Lexically Grounded Subword Segmentation","created":"2024-06-19","cited_by_count":13,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2508.08424","title":"Rethinking Tokenization for Rich Morphology: The Dominance of Unigram over BPE and Morphological Alignment","created":"2025-08-11","cited_by_count":9,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2404.00397","title":"An Analysis of BPE Vocabulary Trimming in Neural Machine Translation","created":"2024-03-30","cited_by_count":7,"area":null},{"id":"2504.10335","title":"MorphTok: Morphologically Grounded Tokenization for Indian Languages","created":"2025-04-14","cited_by_count":7,"area":null},{"id":"2508.15390","title":"Exploiting Vocabulary Frequency Imbalance in Language Model Pre-training","created":"2025-08-21","cited_by_count":7,"area":null},{"id":"2503.14433","title":"Splintering Nonconcatenative Languages for Better Tokenization","created":"2025-03-18","cited_by_count":6,"area":null},{"id":"2309.08351","title":"Headless Language Models: Learning without Predicting with Contrastive Weight Tying","created":"2023-09-15","cited_by_count":5,"area":null},{"id":"2412.11940","title":"The Impact of Token Granularity on the Predictive Power of Language Model Surprisal","created":"2024-12-16","cited_by_count":5,"area":null},{"id":"2606.27019","title":"MinGram: A Minimalist Unigram Tokenizer with High Compression and Competitive Morphological Alignment","created":"2026-06-25","cited_by_count":5,"area":null},{"id":"2309.17339","title":"Scaling Experiments in Self-Supervised Cross-Table Representation Learning","created":"2023-09-29","cited_by_count":4,"area":null},{"id":"2408.11443","title":"Distributional Properties of Subword Regularization","created":"2024-08-21","cited_by_count":4,"area":null},{"id":"2511.03237","title":"MUTANT: A Recipe for Multilingual Tokenizer Design","created":"2025-11-05","cited_by_count":4,"area":null},{"id":"2506.07541","title":"Bit-level BPE: Below the byte boundary","created":"2025-06-09","cited_by_count":3,"area":null},{"id":"2510.09947","title":"Beyond Fertility: Analyzing STRR as a Metric for Multilingual Tokenization Evaluation","created":"2025-10-11","cited_by_count":3,"area":null},{"id":"2511.05578","title":"UTF-8 Plumbing: Byte-level Tokenizers Unavoidably Enable LLMs to Generate Ill-formed UTF-8","created":"2025-11-05","cited_by_count":3,"area":null},{"id":"2602.04289","title":"Proxy Compression for Language Modeling","created":"2026-02-04","cited_by_count":3,"area":null},{"id":"2506.17789","title":"Multilingual Tokenization through the Lens of Indian Languages: Challenges and Insights","created":"2025-06-21","cited_by_count":2,"area":null},{"id":"2601.11643","title":"Syllabic Agglutinative Tokenizations for Indonesian LLM: A Study from Gasing Literacy Learning System","created":"2026-01-14","cited_by_count":2,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2503.04064","title":"Uncovering inequalities in new knowledge learning by large language models across different languages","created":"2025-03-06","cited_by_count":1,"area":null},{"id":"2504.07989","title":"Regional Tiny Stories: Using Small Models to Compare Language Learning and Tokenizer Performance","created":"2025-04-07","cited_by_count":1,"area":null},{"id":"2506.18639","title":"ByteSpan: Information-Driven Subword Tokenisation","created":"2025-06-23","cited_by_count":1,"area":null},{"id":"2511.20849","title":"Length-MAX Tokenizer for Language Models","created":"2025-11-25","cited_by_count":1,"area":null},{"id":"2601.18536","title":"Evaluating Morphological Plausibility of Subword Tokenization via Statistical Alignment with Morpho-Syntactic Features","created":"2026-01-26","cited_by_count":1,"area":null},{"id":"2604.05192","title":"Faster Superword Tokenization","created":"2026-04-06","cited_by_count":1,"area":null},{"id":"2605.13485","title":"Effective Context in Transformers: An Analysis of Fragmentation and Tokenization","created":"2026-05-13","cited_by_count":1,"area":null},{"id":"2401.16055","title":"Stolen Subwords: Importance of Vocabularies for Machine Translation Model Stealing","created":"2024-01-29","cited_by_count":0,"area":null},{"id":"2410.17094","title":"Team Ryu's Submission to SIGMORPHON 2024 Shared Task on Subword Tokenization","created":"2024-10-19","cited_by_count":0,"area":null},{"id":"2602.24174","title":"Task-Centric Acceleration of Small-Language Models","created":"2026-02-27","cited_by_count":0,"area":null},{"id":"2606.15521","title":"Emergent retokenization symmetry in large language models: phenomenology and applications","created":"2026-06-14","cited_by_count":0,"area":null},{"id":"2607.05691","title":"Where to cut, how deep: BPE and Unigram-LM on chemistry SMILES","created":"2026-07-06","cited_by_count":0,"area":null},{"id":"2607.06054","title":"BlueMagpie-TTS: A Token-Efficient Tokenizer, Language Model, and TTS for Taiwanese-Accent Code-Switching Speech","created":"2026-07-07","cited_by_count":0,"area":null},{"id":"2607.17689","title":"Tokenizing Crosslingual Homographs","created":"2026-07-20","cited_by_count":0,"area":null},{"id":"2608.17325","title":"What Tokens are Learned when Tokenization is Optimized Jointly with Language Modeling?","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2608.18025","title":"How Far Should Tokenization Go? Predictive Effectiveness and Relational Losslessness","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2609.00378","title":"Removable and Irreducible: A Token-Cost Ledger for the Multilingual Tokenization Tax","created":"2026-07-17","cited_by_count":0,"area":null},{"id":"2609.12960","title":"Fewer Words, Not Fewer Tokens: Measuring the Sanskrit Tokenization Penalty per Proposition","created":"2026-09-11","cited_by_count":0,"area":null},{"id":"2609.21362","title":"Beyond Atomic Tokens: Factorizing Syllables for Language Model Pretraining","created":"2026-09-18","cited_by_count":0,"area":null},{"id":"2609.35869","title":"The Price of Token Boundaries: Compression Certificates and Prediction","created":"2026-09-26","cited_by_count":0,"area":null}],"next_cursor":null}