{"total":61,"rows":[{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2308.16149","title":"Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models","created":"2023-08-30","cited_by_count":69,"area":null},{"id":"2402.01035","title":"Getting the most out of your tokenizer for pre-training and domain adaptation","created":"2024-02-01","cited_by_count":66,"area":null},{"id":"2306.11372","title":"Democratizing LLMs for Low-Resource Languages by Leveraging their English Dominant Abilities with Linguistically-Diverse Prompts","created":"2023-06-20","cited_by_count":45,"area":null},{"id":"2403.10691","title":"MYTE: Morphology-Driven Byte Encoding for Better and Fairer Multilingual Language Modeling","created":"2024-03-15","cited_by_count":40,"area":null},{"id":"2508.04796","title":"Parity-Aware Byte-Pair Encoding: Improving Cross-lingual Fairness in Tokenization","created":"2025-08-06","cited_by_count":20,"area":null},{"id":"2312.11779","title":"Tokenization Matters: Navigating Data-Scarce Tokenization for Gender Inclusive Language Technologies","created":"2023-12-19","cited_by_count":16,"area":null},{"id":"2502.20273","title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","created":"2025-02-27","cited_by_count":15,"area":null},{"id":"2406.13560","title":"Lexically Grounded Subword Segmentation","created":"2024-06-19","cited_by_count":13,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2509.18750","title":"False Friends Are Not Foes: Investigating Vocabulary Overlap in Multilingual Language Models","created":"2025-09-23","cited_by_count":11,"area":null},{"id":"2601.07220","title":"The Roots of Performance Disparity in Multilingual Language Models: Intrinsic Modeling Difficulty or Design Choices?","created":"2026-01-12","cited_by_count":11,"area":null},{"id":"2502.06468","title":"Beyond Literal Token Overlap: Token Alignability for Multilinguality","created":"2025-02-10","cited_by_count":9,"area":null},{"id":"2510.18840","title":"See the Text: From Tokenization to Visual Reading","created":"2025-10-21","cited_by_count":9,"area":null},{"id":"2510.21909","title":"Explaining and Mitigating Crosslingual Tokenizer Inequities","created":"2025-10-24","cited_by_count":7,"area":null},{"id":"2405.20512","title":"How Multilingual Are Large Language Models Fine-Tuned for Translation?","created":"2024-05-30","cited_by_count":6,"area":null},{"id":"2410.16257","title":"Elucidating the design space of language models for image generation","created":"2024-10-21","cited_by_count":6,"area":null},{"id":"2501.06246","title":"A Partition Cover Approach to Tokenization","created":"2025-01-08","cited_by_count":5,"area":null},{"id":"2602.11174","title":"The Script Tax: Measuring Tokenization-Driven Efficiency and Latency Disparities in Multilingual Language Models","created":"2026-01-19","cited_by_count":4,"area":null},{"id":"2605.01188","title":"Compute Optimal Tokenization","created":"2026-05-02","cited_by_count":4,"area":null},{"id":"2407.02062","title":"Are Data Augmentation Methods in Named Entity Recognition Applicable for Uncertainty Estimation?","created":"2024-07-02","cited_by_count":3,"area":null},{"id":"2608.25089","title":"Apples to Apples? Towards Comparable Crosslingual Language Model Evaluation","created":"2026-08-25","cited_by_count":3,"area":null},{"id":"2410.12989","title":"Qtok: A Comprehensive Framework for Evaluating Multilingual Tokenizer Quality in Large Language Models","created":"2024-10-16","cited_by_count":2,"area":null},{"id":"2506.17789","title":"Multilingual Tokenization through the Lens of Indian Languages: Challenges and Insights","created":"2025-06-21","cited_by_count":2,"area":null},{"id":"2510.06250","title":"Scalable multilingual PII annotation for responsible AI in LLMs","created":"2025-10-03","cited_by_count":2,"area":null},{"id":"2510.10827","title":"Happiness is Sharing a Vocabulary: A Study of Transliteration Methods","created":"2025-10-12","cited_by_count":2,"area":null},{"id":"2510.12389","title":"Tokenization Disparities as Infrastructure Bias: How Subword Systems Create Inequities in LLM Access and Efficiency","created":"2025-10-14","cited_by_count":2,"area":null},{"id":"2601.10580","title":"Form and Meaning in Intrinsic Multilingual Evaluations","created":"2026-01-15","cited_by_count":2,"area":null},{"id":"2607.26831","title":"Language Models are not Equally Robust to Non-Canonical Tokenization across Languages","created":"2026-07-29","cited_by_count":2,"area":null},{"id":"2608.18062","title":"TokEval: A Tokenizer Evaluation Suite","created":"2026-08-18","cited_by_count":2,"area":null},{"id":"2608.26449","title":"Vowel Signs Are Not Letters: A Pre-tokenization Ceiling on Multilingual Tokenizer Fertility","created":"2026-08-26","cited_by_count":2,"area":null},{"id":"2502.09645","title":"From No to Know: Taxonomy, Challenges, and Opportunities for Negation Understanding in Multimodal Foundation Models","created":"2025-02-10","cited_by_count":1,"area":null},{"id":"2503.04064","title":"Uncovering inequalities in new knowledge learning by large language models across different languages","created":"2025-03-06","cited_by_count":1,"area":null},{"id":"2503.04174","title":"UniNet: A Unified Multi-granular Traffic Modeling Framework for Network Security","created":"2025-03-06","cited_by_count":1,"area":null},{"id":"2510.17001","title":"Vocab Diet: Reshaping the Vocabulary of LLMs via Vector Arithmetic","created":"2025-10-19","cited_by_count":1,"area":null},{"id":"2601.18536","title":"Evaluating Morphological Plausibility of Subword Tokenization via Statistical Alignment with Morpho-Syntactic Features","created":"2026-01-26","cited_by_count":1,"area":null},{"id":"2602.13194","title":"Semantic Chunking and the Entropy of Natural Language","created":"2026-02-13","cited_by_count":1,"area":null},{"id":"2603.24413","title":"Pingala: Prosody-Aware Decoding for Sanskrit Poetry Generation","created":"2026-03-25","cited_by_count":1,"area":null},{"id":"2604.17633","title":"Copy First, Translate Later: Interpreting Translation Dynamics in Multilingual Pretraining","created":"2026-04-19","cited_by_count":1,"area":null},{"id":"2507.03378","title":"Making Sense of Korean Sentences: A Comprehensive Evaluation of LLMs through KoSEnd Dataset","created":"2025-07-04","cited_by_count":0,"area":null},{"id":"2507.07824","title":"Conditional Unigram Tokenization with Parallel Data","created":"2025-07-10","cited_by_count":0,"area":null},{"id":"2509.08812","title":"MoVoC: Morphology-Aware Subword Construction for Geez Script Languages","created":"2025-09-10","cited_by_count":0,"area":null},{"id":"2510.27254","title":"Languages are Modalities: Cross-Lingual Alignment via Encoder Injection","created":"2025-10-31","cited_by_count":0,"area":null},{"id":"2601.02867","title":"Training Language Models with homotokens Leads to Delayed Overfitting","created":"2026-01-06","cited_by_count":0,"area":null},{"id":"2602.04241","title":"Tokenization and Morphological Fidelity in Uralic NLP: A Cross-Lingual Evaluation","created":"2026-02-04","cited_by_count":0,"area":null},{"id":"2602.06973","title":"Does Visual Rendering Bypass Tokenization? Investigating Script-Tokenizer Misalignment in Pixel-Based Language Models","created":"2026-01-12","cited_by_count":0,"area":null},{"id":"2603.22629","title":"LGSE: Lexically Grounded Subword Embedding Initialization for Low-Resource Language Adaptation","created":"2026-03-23","cited_by_count":0,"area":null},{"id":"2603.26013","title":"Toward Culturally Grounded Natural Language Processing","created":"2026-03-27","cited_by_count":0,"area":null},{"id":"2605.26683","title":"An In-Vitro Study on Cross-Lingual Generalization in Language Models","created":"2026-05-26","cited_by_count":0,"area":null},{"id":"2606.06738","title":"Modular Monolingual Adaptation using Pretrained Language Models","created":"2026-06-04","cited_by_count":0,"area":null},{"id":"2606.23566","title":"LangMAP: A Language-Adaptive Approach to Tokenization","created":"2026-06-22","cited_by_count":0,"area":null},{"id":"2607.08362","title":"Echoes Across Vietnam's Highlands, Delta, and Coast: A Multilingual Corpus for Cham, Khmer, and Tay-Nung","created":"2026-07-09","cited_by_count":0,"area":null},{"id":"2607.15209","title":"Expanding the Lexicon of Ge'ez Based African Languages: A Comparative Study of Amharic and Tigrinya","created":"2026-07-16","cited_by_count":0,"area":null},{"id":"2607.16117","title":"Rate-Utility Frontiers for Language Encodings: Comparing Tokens, Bytes, and Pixels Under Controlled Linguistic Content","created":"2026-07-17","cited_by_count":0,"area":null},{"id":"2607.17689","title":"Tokenizing Crosslingual Homographs","created":"2026-07-20","cited_by_count":0,"area":null},{"id":"2607.23319","title":"BHARATI: Morphology-Aware Tokenizers for Classical Indian Languages with Subword Fertility Analysis","created":"2026-07-25","cited_by_count":0,"area":null},{"id":"2608.03446","title":"Predicting Multilingual Classification and Translation Performance of LLMs with Cross-Lingual Alignment -- Is English Enough?","created":"2026-08-04","cited_by_count":0,"area":null},{"id":"2608.25904","title":"One Form to Transfer Them All: Pretraining Multilingual Language Models Beyond Native Orthography","created":"2026-08-26","cited_by_count":0,"area":null},{"id":"2608.28924","title":"Causal Interventions Reveal Typologically Organized Syntactic Mechanisms in Multilingual Language Models","created":"2026-08-28","cited_by_count":0,"area":null},{"id":"2609.12960","title":"Fewer Words, Not Fewer Tokens: Measuring the Sanskrit Tokenization Penalty per Proposition","created":"2026-09-11","cited_by_count":0,"area":null}],"next_cursor":null}