{"total":180,"rows":[{"id":"2307.10169","title":"Challenges and Applications of Large Language Models","created":"2023-07-19","cited_by_count":254,"area":null},{"id":"2305.13707","title":"Do All Languages Cost the Same? Tokenization in the Era of Commercial Language Models","created":"2023-05-23","cited_by_count":192,"area":null},{"id":"2310.08754","title":"Tokenizer Choice For LLM Training: Negligible or Crucial?","created":"2023-10-12","cited_by_count":108,"area":null},{"id":"2309.16575","title":"A Benchmark for Learning to Translate a New Language from One Grammar Book","created":"2023-09-28","cited_by_count":75,"area":null},{"id":"2310.16787","title":"The Data Provenance Initiative: A Large Scale Audit of Dataset Licensing & Attribution in AI","created":"2023-10-25","cited_by_count":73,"area":null},{"id":"2507.07955","title":"Dynamic Chunking for End-to-End Hierarchical Sequence Modeling","created":"2025-07-10","cited_by_count":72,"area":null},{"id":"2308.16149","title":"Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models","created":"2023-08-30","cited_by_count":69,"area":null},{"id":"2411.14198","title":"Why do language models perform worse for morphologically complex languages?","created":"2024-11-21","cited_by_count":62,"area":null},{"id":"2410.05864","title":"From Tokens to Words: On the Inner Lexicon of LLMs","created":"2024-10-08","cited_by_count":48,"area":null},{"id":"2408.04303","title":"Trans-Tokenization and Cross-lingual Vocabulary Transfers: Language Adaptation of LLMs for Low-Resource NLP","created":"2024-08-08","cited_by_count":36,"area":null},{"id":"2404.14408","title":"SpaceByte: Towards Deleting Tokenization from Large Language Modeling","created":"2024-04-22","cited_by_count":34,"area":null},{"id":"2309.12244","title":"ChaCha: Leveraging Large Language Models to Prompt Children to Share Their Emotions about Personal Events","created":"2023-09-21","cited_by_count":31,"area":null},{"id":"2403.00025","title":"On the Challenges and Opportunities in Generative AI","created":"2024-02-28","cited_by_count":31,"area":null},{"id":"2405.07883","title":"Zero-Shot Tokenizer Transfer","created":"2024-05-13","cited_by_count":31,"area":null},{"id":"2407.08818","title":"MAGNET: Improving the Multilingual Fairness of Language Models with Adaptive Gradient-Based Tokenization","created":"2024-07-11","cited_by_count":31,"area":null},{"id":"2310.11761","title":"A Comprehensive Evaluation of Large Language Models on Legal Judgment Prediction","created":"2023-10-18","cited_by_count":28,"area":null},{"id":"2403.03814","title":"Evaluating the Elementary Multilingual Capabilities of Large Language Models with MultiQ","created":"2024-03-06","cited_by_count":27,"area":null},{"id":"2403.00686","title":"A Bit of a Problem: Measurement Disparities in Dataset Sizes Across Languages","created":"2024-03-01","cited_by_count":25,"area":null},{"id":"2406.11477","title":"How Can We Effectively Expand the Vocabulary of LLMs with 0.01GB of Target Language Text?","created":"2024-06-17","cited_by_count":22,"area":null},{"id":"2410.03730","title":"Teuken-7B-Base & Teuken-7B-Instruct: Towards European LLMs","created":"2024-09-30","cited_by_count":22,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2408.08541","title":"Where is the signal in tokenization space?","created":"2024-08-16","cited_by_count":21,"area":null},{"id":"2407.01264","title":"SignCLIP: Connecting Text and Sign Language by Contrastive Learning","created":"2024-07-01","cited_by_count":20,"area":null},{"id":"2409.11501","title":"Egalitarian Language Representation in Language Models: It All Begins with Tokenizers","created":"2024-09-17","cited_by_count":20,"area":null},{"id":"2402.10712","title":"An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Language Model Inference","created":"2024-02-16","cited_by_count":18,"area":null},{"id":"2404.01856","title":"Poro 34B and the Blessing of Multilinguality","created":"2024-04-02","cited_by_count":18,"area":null},{"id":"2404.01954","title":"HyperCLOVA X Technical Report","created":"2024-04-02","cited_by_count":18,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2402.09949","title":"Multi-word Tokenization for Sequence Compression","created":"2024-02-15","cited_by_count":17,"area":null},{"id":"2503.21841","title":"HyperFree: A Channel-adaptive and Tuning-free Foundation Model for Hyperspectral Remote Sensing Imagery","created":"2025-03-27","cited_by_count":17,"area":null},{"id":"2309.13320","title":"GlotScript: A Resource and Tool for Low Resource Writing System Identification","created":"2023-09-23","cited_by_count":15,"area":null},{"id":"2402.14714","title":"Efficient and Effective Vocabulary Expansion Towards Multilingual Large Language Models","created":"2024-02-22","cited_by_count":15,"area":null},{"id":"2509.05486","title":"The Token Tax: Systematic Bias in Multilingual Tokenization","created":"2025-09-05","cited_by_count":14,"area":null},{"id":"2404.17047","title":"Near to Mid-term Risks and Opportunities of Open-Source Generative AI","created":"2024-04-25","cited_by_count":13,"area":null},{"id":"2405.08597","title":"Risks and Opportunities of Open-Source Generative AI","created":"2024-05-14","cited_by_count":13,"area":null},{"id":"2407.07080","title":"Adapting LLMs to Hebrew: Unveiling DictaLM 2.0 with Enhanced Vocabulary and Instruction Capabilities","created":"2024-07-09","cited_by_count":13,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2505.21627","title":"Is Your LLM Overcharging You? Tokenization, Transparency, and Incentives","created":"2025-05-27","cited_by_count":13,"area":null},{"id":"2405.13929","title":"Vikhr: The Family of Open-Source Instruction-Tuned Large Language Models for Russian","created":"2024-05-22","cited_by_count":12,"area":null},{"id":"2407.05022","title":"A Principled Framework for Evaluating on Typologically Diverse Languages","created":"2024-07-06","cited_by_count":12,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2509.18750","title":"False Friends Are Not Foes: Investigating Vocabulary Overlap in Multilingual Language Models","created":"2025-09-23","cited_by_count":11,"area":null},{"id":"2601.07220","title":"The Roots of Performance Disparity in Multilingual Language Models: Intrinsic Modeling Difficulty or Design Choices?","created":"2026-01-12","cited_by_count":11,"area":null},{"id":"2309.16573","title":"Language Models as a Service: Overview of a New Paradigm and its Challenges","created":"2023-09-28","cited_by_count":10,"area":null},{"id":"2402.13703","title":"Investigating Multilingual Instruction-Tuning: Do Polyglot Models Demand for Multilingual Instructions?","created":"2024-02-21","cited_by_count":10,"area":null},{"id":"2412.15210","title":"Tokenisation is NP-Complete","created":"2024-12-19","cited_by_count":10,"area":null},{"id":"2504.06011","title":"Llama-3-Nanda-10B-Chat: An Open Generative Large Language Model for Hindi","created":"2025-04-08","cited_by_count":10,"area":null},{"id":"2403.03923","title":"Did Translation Models Get More Robust Without Anyone Even Noticing?","created":"2024-03-06","cited_by_count":9,"area":null},{"id":"2405.05966","title":"Natural Language Processing RELIES on Linguistics","created":"2024-05-09","cited_by_count":9,"area":null},{"id":"2410.09644","title":"Adapters for Altering LLM Vocabularies: What Languages Benefit the Most?","created":"2024-10-12","cited_by_count":9,"area":null},{"id":"2507.23248","title":"Script Fragmentation and Format: What Drives the English-Bengali Performance Gap in Open LLMs?","created":"2025-07-31","cited_by_count":9,"area":null},{"id":"2509.20045","title":"Tokenization and Representation Biases in Multilingual Models on Dialectal NLP Tasks","created":"2025-09-24","cited_by_count":9,"area":null},{"id":"2401.10660","title":"Accelerating Multilingual Language Model for Excessively Tokenized Languages","created":"2024-01-19","cited_by_count":8,"area":null},{"id":"2404.05783","title":"A Survey on Responsible Generative AI: What to Generate and What Not","created":"2024-04-08","cited_by_count":8,"area":null},{"id":"2412.10244","title":"Efficient Continual Pre-training of LLMs for Low-resource Languages","created":"2024-12-13","cited_by_count":8,"area":null},{"id":"2506.01840","title":"Minimal Pair-Based Evaluation of Code-Switching","created":"2025-06-02","cited_by_count":8,"area":null},{"id":"2507.12720","title":"FLEXITOKENS: Flexible Tokenization for Evolving Language Models","created":"2025-07-17","cited_by_count":8,"area":null},{"id":"2403.05434","title":"Cost-Performance Optimization for Processing Low-Resource Language Tasks Using Commercial LLMs","created":"2024-03-08","cited_by_count":7,"area":null},{"id":"2501.04473","title":"When LLMs Struggle: Reference-less Translation Evaluation for Low-resource Languages","created":"2025-01-08","cited_by_count":7,"area":null},{"id":"2505.18951","title":"BnMMLU: Measuring Massive Multitask Language Understanding in Bengali","created":"2025-05-25","cited_by_count":7,"area":null},{"id":"2507.22445","title":"AI-generated stories favour stability over change: homogeneity and cultural stereotyping in narratives generated by gpt-4o-mini","created":"2025-07-30","cited_by_count":7,"area":null},{"id":"2510.21909","title":"Explaining and Mitigating Crosslingual Tokenizer Inequities","created":"2025-10-24","cited_by_count":7,"area":null},{"id":"2405.20512","title":"How Multilingual Are Large Language Models Fine-Tuned for Translation?","created":"2024-05-30","cited_by_count":6,"area":null},{"id":"2412.11704","title":"Adapting Chat Language Models Using Only Target Unlabeled Language Data","created":"2024-12-16","cited_by_count":6,"area":null},{"id":"2501.16516","title":"How well can LLMs Grade Essays in Arabic?","created":"2025-01-27","cited_by_count":6,"area":null},{"id":"2508.19089","title":"It's All About In-Context Learning! Teaching Extremely Low-Resource Languages to LLMs","created":"2025-08-26","cited_by_count":6,"area":null},{"id":"2509.11570","title":"Bhaasha, Bhasa, Zaban: A Survey for Low-Resourced Languages in South Asia -- Current Stage and Challenges","created":"2025-09-15","cited_by_count":6,"area":null},{"id":"2601.13260","title":"Stop Taking Tokenizers for Granted: They Are Core Design Decisions in Large Language Models","created":"2026-01-19","cited_by_count":6,"area":null},{"id":"2409.09662","title":"ExploreSelf: Fostering User-driven Exploration and Reflection on Personal Challenges with Adaptive Guidance by Large Language Models","created":"2024-09-15","cited_by_count":5,"area":null},{"id":"2506.01084","title":"zip2zip: Inference-Time Adaptive Tokenization via Online Compression","created":"2025-06-01","cited_by_count":5,"area":null},{"id":"2602.16201","title":"Long-Tail Knowledge in Large Language Models: Taxonomy, Mechanisms, Interventions and Implications","created":"2026-02-18","cited_by_count":5,"area":null},{"id":"2310.11029","title":"Core Building Blocks: Next Gen Geo Spatial GPT Application","created":"2023-10-17","cited_by_count":4,"area":null},{"id":"2406.09559","title":"Decoding the Diversity: A Review of the Indic AI Research Landscape","created":"2024-06-13","cited_by_count":4,"area":null},{"id":"2503.01493","title":"Sherkala-Chat: Building a State-of-the-Art LLM for Kazakh in a Moderately Resourced Setting","created":"2025-03-03","cited_by_count":4,"area":null},{"id":"2505.18638","title":"Multilingual Question Answering in Low-Resource Settings: A Dzongkha-English Benchmark for Foundation Models","created":"2025-05-24","cited_by_count":4,"area":null},{"id":"2510.05181","title":"Auditing Pay-Per-Token in Large Language Models","created":"2025-10-05","cited_by_count":4,"area":null},{"id":"2511.03237","title":"MUTANT: A Recipe for Multilingual Tokenizer Design","created":"2025-11-05","cited_by_count":4,"area":null},{"id":"2605.01188","title":"Compute Optimal Tokenization","created":"2026-05-02","cited_by_count":4,"area":null},{"id":"2605.14890","title":"Tokenizer Fertility and Zero-Shot Performance of Foundation Models on Ukrainian Legal Text: A Comparative Study","created":"2026-05-14","cited_by_count":4,"area":null},{"id":"2605.24718","title":"The Tokenizer Tax Across 25 European Languages: Domain Invariance, Cross-Lingual Few-Shot Effects, and the Ukrainian Penalty","created":"2026-05-23","cited_by_count":4,"area":null},{"id":"2407.07566","title":"HebDB: a Weakly Supervised Dataset for Hebrew Speech Processing","created":"2024-07-10","cited_by_count":3,"area":null},{"id":"2506.07541","title":"Bit-level BPE: Below the byte boundary","created":"2025-06-09","cited_by_count":3,"area":null},{"id":"2509.14405","title":"Adding LLMs to the psycholinguistic norming toolbox: A practical guide to getting the most out of human ratings","created":"2025-09-17","cited_by_count":3,"area":null},{"id":"2511.05578","title":"UTF-8 Plumbing: Byte-level Tokenizers Unavoidably Enable LLMs to Generate Ill-formed UTF-8","created":"2025-11-05","cited_by_count":3,"area":null},{"id":"2602.04289","title":"Proxy Compression for Language Modeling","created":"2026-02-04","cited_by_count":3,"area":null},{"id":"2602.14653","title":"Is Information Density Uniform when Utterances are Grounded on Perception and Discourse?","created":"2026-02-16","cited_by_count":3,"area":null},{"id":"2604.04204","title":"How Does \"English (US)\" Become the Default? Triangulating Structural Bias Towards American English Across the LLM Pipeline","created":"2026-04-05","cited_by_count":3,"area":null},{"id":"2605.08044","title":"Fast Byte Latent Transformer","created":"2026-05-08","cited_by_count":3,"area":null},{"id":"2608.25089","title":"Apples to Apples? Towards Comparable Crosslingual Language Model Evaluation","created":"2026-08-25","cited_by_count":3,"area":null},{"id":"2406.09140","title":"Investigating the translation capabilities of Large Language Models trained on parallel data only","created":"2024-06-13","cited_by_count":2,"area":null},{"id":"2410.03568","title":"Towards Linguistically-Aware and Language-Independent Tokenization for Large Language Models (LLMs)","created":"2024-10-04","cited_by_count":2,"area":null},{"id":"2411.15397","title":"Visual-Word Tokenizer: Beyond Fixed Sets of Tokens in Vision Transformers","created":"2024-11-23","cited_by_count":2,"area":null},{"id":"2502.11417","title":"DiSCo: Device-Server Collaborative LLM-Based Text Streaming Services","created":"2025-02-17","cited_by_count":2,"area":null},{"id":"2504.00289","title":"Do Chinese models speak Chinese languages?","created":"2025-03-31","cited_by_count":2,"area":null},{"id":"2506.01535","title":"Dictionaries to the Rescue: Cross-Lingual Vocabulary Transfer for Low-Resource Languages Using Bilingual Dictionaries","created":"2025-06-02","cited_by_count":2,"area":null},{"id":"2506.15138","title":"Less Is More: Reducing Token Counts Without Compromising Performance","created":"2025-06-18","cited_by_count":2,"area":null},{"id":"2506.17789","title":"Multilingual Tokenization through the Lens of Indian Languages: Challenges and Insights","created":"2025-06-21","cited_by_count":2,"area":null},{"id":"2507.16323","title":"SpeLLM: Character-Level Multi-Head Decoding","created":"2025-07-22","cited_by_count":2,"area":null},{"id":"2509.25516","title":"Beyond WER: Probing Whisper's Sub-token Decoder Across Diverse Language Resource Levels","created":"2025-09-29","cited_by_count":2,"area":null}],"next_cursor":"100"}