{"total":38,"rows":[{"id":"2404.16816","title":"IndicGenBench: A Multilingual Benchmark to Evaluate Generation Capabilities of LLMs on Indic Languages","created":"2024-04-25","cited_by_count":56,"area":null},{"id":"2404.14408","title":"SpaceByte: Towards Deleting Tokenization from Large Language Modeling","created":"2024-04-22","cited_by_count":34,"area":null},{"id":"2407.08818","title":"MAGNET: Improving the Multilingual Fairness of Language Models with Adaptive Gradient-Based Tokenization","created":"2024-07-11","cited_by_count":31,"area":null},{"id":"2508.04796","title":"Parity-Aware Byte-Pair Encoding: Improving Cross-lingual Fairness in Tokenization","created":"2025-08-06","cited_by_count":20,"area":null},{"id":"2506.19004","title":"Broken Tokens? Your Language Model can Secretly Handle Non-Canonical Tokenizations","created":"2025-06-23","cited_by_count":17,"area":null},{"id":"2411.18553","title":"Retrofitting Large Language Models with Dynamic Tokenization","created":"2024-11-27","cited_by_count":16,"area":null},{"id":"2410.09303","title":"Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles","created":"2024-10-11","cited_by_count":15,"area":null},{"id":"2511.01380","title":"Confounding Factors in Relating Model Performance to Morphology","created":"2025-11-03","cited_by_count":14,"area":null},{"id":"2407.16607","title":"Data Mixture Inference: What do BPE Tokenizers Reveal about their Training Data?","created":"2024-07-23","cited_by_count":13,"area":null},{"id":"2505.24689","title":"BPE Stays on SCRIPT: Structured Encoding for Robust Multilingual Pretokenization","created":"2025-05-30","cited_by_count":12,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2601.07220","title":"The Roots of Performance Disparity in Multilingual Language Models: Intrinsic Modeling Difficulty or Design Choices?","created":"2026-01-12","cited_by_count":11,"area":null},{"id":"2405.05966","title":"Natural Language Processing RELIES on Linguistics","created":"2024-05-09","cited_by_count":9,"area":null},{"id":"2502.06468","title":"Beyond Literal Token Overlap: Token Alignability for Multilinguality","created":"2025-02-10","cited_by_count":9,"area":null},{"id":"2512.15586","title":"Bolmo: Byteifying the Next Generation of Language Models","created":"2025-12-17","cited_by_count":9,"area":null},{"id":"2507.12720","title":"FLEXITOKENS: Flexible Tokenization for Evolving Language Models","created":"2025-07-17","cited_by_count":8,"area":null},{"id":"2410.23684","title":"Improbable Bigrams Expose Vulnerabilities of Incomplete Tokens in Byte-Level Tokenizers","created":"2024-10-31","cited_by_count":7,"area":null},{"id":"2510.21909","title":"Explaining and Mitigating Crosslingual Tokenizer Inequities","created":"2025-10-24","cited_by_count":7,"area":null},{"id":"2501.06246","title":"A Partition Cover Approach to Tokenization","created":"2025-01-08","cited_by_count":5,"area":null},{"id":"2605.01188","title":"Compute Optimal Tokenization","created":"2026-05-02","cited_by_count":4,"area":null},{"id":"2511.05578","title":"UTF-8 Plumbing: Byte-level Tokenizers Unavoidably Enable LLMs to Generate Ill-formed UTF-8","created":"2025-11-05","cited_by_count":3,"area":null},{"id":"2602.04289","title":"Proxy Compression for Language Modeling","created":"2026-02-04","cited_by_count":3,"area":null},{"id":"2608.25089","title":"Apples to Apples? Towards Comparable Crosslingual Language Model Evaluation","created":"2026-08-25","cited_by_count":3,"area":null},{"id":"2601.10580","title":"Form and Meaning in Intrinsic Multilingual Evaluations","created":"2026-01-15","cited_by_count":2,"area":null},{"id":"2604.16656","title":"Defragmenting Language Models: An Interpretability-based Approach for Vocabulary Expansion","created":"2026-04-17","cited_by_count":2,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2606.20993","title":"Phonemes to the Rescue: Multilingual Tokenization Based on International Phonetic Alphabet","created":"2026-06-18","cited_by_count":2,"area":null},{"id":"2506.18639","title":"ByteSpan: Information-Driven Subword Tokenisation","created":"2025-06-23","cited_by_count":1,"area":null},{"id":"2510.06461","title":"Linguistically Informed Tokenization Improves ASR for Underresourced Languages","created":"2025-10-07","cited_by_count":1,"area":null},{"id":"2605.29459","title":"Kronecker Embeddings: Byte-Level Structured Token Representations for Parameter-Efficient Language Models","created":"2026-05-28","cited_by_count":1,"area":null},{"id":"2411.01474","title":"MoCE: Adaptive Mixture of Contextualization Experts for Byte-based Neural Machine Translation","created":"2024-11-03","cited_by_count":0,"area":null},{"id":"2504.01540","title":"From Sm{\\o}r-re-br{\\o}d to Subwords: Training LLMs on Danish, One Morpheme at a Time","created":"2025-04-02","cited_by_count":0,"area":null},{"id":"2606.14122","title":"Beyond Perplexity: UTF-8 Validity in Byte-aware Language Models","created":"2026-06-12","cited_by_count":0,"area":null},{"id":"2606.15044","title":"Equity with Efficiency: An Empirical Study of Tokenizers for Multilingual Large Language Models","created":"2026-06-13","cited_by_count":0,"area":null},{"id":"2606.15521","title":"Emergent retokenization symmetry in large language models: phenomenology and applications","created":"2026-06-14","cited_by_count":0,"area":null},{"id":"2608.15454","title":"Dynamic Multi-Byte Prediction With Hierarchical Language Models","created":"2026-08-16","cited_by_count":0,"area":null},{"id":"2608.17325","title":"What Tokens are Learned when Tokenization is Optimized Jointly with Language Modeling?","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2609.15528","title":"To Each Language Its Tokenizer: Modular Tokenizers for Efficient Multilingual LLMs","created":"2026-09-14","cited_by_count":0,"area":null}],"next_cursor":null}