{"total":59,"rows":[{"id":"2503.13423","title":"SuperBPE: Space Travel for Language Models","created":"2025-03-17","cited_by_count":63,"area":null},{"id":"2411.14198","title":"Why do language models perform worse for morphologically complex languages?","created":"2024-11-21","cited_by_count":62,"area":null},{"id":"2504.00178","title":"Boundless Byte Pair Encoding: Breaking the Pre-tokenization Barrier","created":"2025-03-31","cited_by_count":32,"area":null},{"id":"2409.04599","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","created":"2024-09-06","cited_by_count":31,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2409.11501","title":"Egalitarian Language Representation in Language Models: It All Begins with Tokenizers","created":"2024-09-17","cited_by_count":20,"area":null},{"id":"2502.18795","title":"Anything Goes? A Crosslinguistic Study of (Im)possible Language Learning in LMs","created":"2025-02-26","cited_by_count":20,"area":null},{"id":"2508.04796","title":"Parity-Aware Byte-Pair Encoding: Improving Cross-lingual Fairness in Tokenization","created":"2025-08-06","cited_by_count":20,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2404.03626","title":"Training LLMs over Neurally Compressed Text","created":"2024-04-04","cited_by_count":17,"area":null},{"id":"2410.09303","title":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","created":"2024-10-11","cited_by_count":15,"area":null},{"id":"2503.19693","title":"AdaptiVocab: Enhancing LLM Efficiency in Focused Domains through Lightweight Vocabulary Adaptation","created":"2025-03-25","cited_by_count":15,"area":null},{"id":"2411.07854","title":"Tucano: Advancing Neural Text Generation for Portuguese","created":"2024-11-12","cited_by_count":13,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2601.07220","title":"The Roots of Performance Disparity in Multilingual Language Models: Intrinsic Modeling Difficulty or Design Choices?","created":"2026-01-12","cited_by_count":11,"area":null},{"id":"2412.15210","title":"Tokenisation is NP-Complete","created":"2024-12-19","cited_by_count":10,"area":null},{"id":"2506.13487","title":"TurBLiMP: A Turkish Benchmark of Linguistic Minimal Pairs","created":"2025-06-16","cited_by_count":9,"area":null},{"id":"2508.08424","title":"Rethinking Tokenization for Rich Morphology: The Dominance of Unigram over BPE and Morphological Alignment","created":"2025-08-11","cited_by_count":9,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2412.06926","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","created":"2024-12-09","cited_by_count":8,"area":null},{"id":"2410.23684","title":"Improbable Bigrams Expose Vulnerabilities of Incomplete Tokens in Byte-Level Tokenizers","created":"2024-10-31","cited_by_count":7,"area":null},{"id":"2412.16642","title":"L3TC: Leveraging RWKV for Learned Lossless Low-Complexity Text Compression","created":"2024-12-21","cited_by_count":7,"area":null},{"id":"2510.21909","title":"Explaining and Mitigating Crosslingual Tokenizer Inequities","created":"2025-10-24","cited_by_count":7,"area":null},{"id":"2502.12560","title":"How does a Language-Specific Tokenizer affect LLMs?","created":"2025-02-18","cited_by_count":6,"area":null},{"id":"2601.13260","title":"Stop Taking Tokenizers for Granted: They Are Core Design Decisions in Large Language Models","created":"2026-01-19","cited_by_count":6,"area":null},{"id":"2603.15644","title":"Tokenization Tradeoffs in Structured EHR Foundation Models","created":"2026-03-03","cited_by_count":6,"area":null},{"id":"2409.15370","title":"Tokenization for Molecular Foundation Models","created":"2024-09-19","cited_by_count":5,"area":null},{"id":"2501.06246","title":"A Partition Cover Approach to Tokenization","created":"2025-01-08","cited_by_count":5,"area":null},{"id":"2507.01785","title":"MuRating: A High Quality Data Selecting Approach to Multilingual Large Language Model Pretraining","created":"2025-07-02","cited_by_count":5,"area":null},{"id":"2511.15709","title":"Tokenisation over Bounded Alphabets is Hard","created":"2025-11-19","cited_by_count":5,"area":null},{"id":"2605.01188","title":"Compute Optimal Tokenization","created":"2026-05-02","cited_by_count":4,"area":null},{"id":"2506.07541","title":"Bit-level BPE: Below the byte boundary","created":"2025-06-09","cited_by_count":3,"area":null},{"id":"2507.22543","title":"Pre-trained Models Perform the Best When Token Distributions Follow Zipf's Law","created":"2025-07-30","cited_by_count":3,"area":null},{"id":"2510.09947","title":"Beyond Fertility: Analyzing STRR as a Metric for Multilingual Tokenization Evaluation","created":"2025-10-11","cited_by_count":3,"area":null},{"id":"2601.21665","title":"AdaptBPE: From General Purpose to Specialized Tokenizers","created":"2026-01-29","cited_by_count":3,"area":null},{"id":"2412.13377","title":"DateLogicQA: Benchmarking Temporal Biases in Large Language Models","created":"2024-12-17","cited_by_count":2,"area":null},{"id":"2506.07833","title":"Improving Large Language Models with Concept-Aware Fine-Tuning","created":"2025-06-09","cited_by_count":2,"area":null},{"id":"2506.15138","title":"Less Is More: Reducing Token Counts Without Compromising Performance","created":"2025-06-18","cited_by_count":2,"area":null},{"id":"2512.03989","title":"Teaching Old Tokenizers New Words: Efficient Tokenizer Adaptation for Pre-trained Models","created":"2025-12-03","cited_by_count":2,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2608.26449","title":"Vowel Signs Are Not Letters: A Pre-tokenization Ceiling on Multilingual Tokenizer Fertility","created":"2026-08-26","cited_by_count":2,"area":null},{"id":"2503.04064","title":"Uncovering inequalities in new knowledge learning by large language models across different languages","created":"2025-03-06","cited_by_count":1,"area":null},{"id":"2508.06621","title":"Train It and Forget It: Merge Lists are Unnecessary for BPE Inference in Language Models","created":"2025-08-08","cited_by_count":1,"area":null},{"id":"2511.20849","title":"Length-MAX Tokenizer for Language Models","created":"2025-11-25","cited_by_count":1,"area":null},{"id":"2512.21913","title":"GQ-VAE: A gated quantized VAE for learning variable length tokens","created":"2025-12-26","cited_by_count":1,"area":null},{"id":"2603.25309","title":"Separate Before You Compress: The WWHO Tokenization Architecture","created":"2026-03-26","cited_by_count":1,"area":null},{"id":"2605.13485","title":"Effective Context in Transformers: An Analysis of Fragmentation and Tokenization","created":"2026-05-13","cited_by_count":1,"area":null},{"id":"2607.23362","title":"Joint Optimization for Greedy Longest-match Tokenization","created":"2026-07-25","cited_by_count":1,"area":null},{"id":"2509.08812","title":"MoVoC: Morphology-Aware Subword Construction for Geez Script Languages","created":"2025-09-10","cited_by_count":0,"area":null},{"id":"2512.11502","title":"Building Patient Journeys in Hebrew: A Language Model for Clinical Timeline Extraction","created":"2025-12-12","cited_by_count":0,"area":null},{"id":"2601.02867","title":"Training Language Models with homotokens Leads to Delayed Overfitting","created":"2026-01-06","cited_by_count":0,"area":null},{"id":"2601.11518","title":"How Long Is a Piece of String? A Brief Empirical Analysis of Tokenizers","created":"2026-01-16","cited_by_count":0,"area":null},{"id":"2601.13588","title":"TREX: Tokenizer Regression for Optimal Data Mixture","created":"2026-01-20","cited_by_count":0,"area":null},{"id":"2607.15209","title":"Expanding the Lexicon of Ge'ez Based African Languages: A Comparative Study of Amharic and Tigrinya","created":"2026-07-16","cited_by_count":0,"area":null},{"id":"2609.01386","title":"When Tokenization is Secretly Output Supervision","created":"2026-09-01","cited_by_count":0,"area":null},{"id":"2609.05022","title":"MoirfEolas and Cr\\'iochScore: Developing Resources for and the Evaluation of Tokenization Alignment with Irish Morphology","created":"2026-09-04","cited_by_count":0,"area":null},{"id":"2609.12960","title":"Fewer Words, Not Fewer Tokens: Measuring the Sanskrit Tokenization Penalty per Proposition","created":"2026-09-11","cited_by_count":0,"area":null},{"id":"2609.15528","title":"To Each Language Its Tokenizer: Modular Tokenizers for Efficient Multilingual LLMs","created":"2026-09-14","cited_by_count":0,"area":null}],"next_cursor":null}