{"total":28,"rows":[{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2307.15703","title":"Uncertainty in Natural Language Generation: From Theory to Applications","created":"2023-07-28","cited_by_count":61,"area":null},{"id":"2207.06991","title":"Language Modelling with Pixels","created":"2022-07-14","cited_by_count":55,"area":null},{"id":"2312.10523","title":"Paloma: A Benchmark for Evaluating Language Model Fit","created":"2023-12-16","cited_by_count":54,"area":null},{"id":"2504.13139","title":"Syntactic and Semantic Control of Large Language Models via Sequential Monte Carlo","created":"2025-04-17","cited_by_count":46,"area":null},{"id":"2206.02608","title":"What do tokens know about their characters and how do they know it?","created":"2022-06-06","cited_by_count":36,"area":null},{"id":"2406.14561","title":"How to Compute the Probability of a Word","created":"2024-06-20","cited_by_count":36,"area":null},{"id":"2412.03719","title":"From Language Models over Tokens to Language Models over Characters","created":"2024-12-04","cited_by_count":31,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2408.08541","title":"Where is the signal in tokenization space?","created":"2024-08-16","cited_by_count":21,"area":null},{"id":"2306.17757","title":"Should you marginalize over possible tokenizations?","created":"2023-06-30","cited_by_count":18,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2506.19004","title":"Broken Tokens? Your Language Model can Secretly Handle Non-Canonical Tokenizations","created":"2025-06-23","cited_by_count":17,"area":null},{"id":"2410.02691","title":"On the Proper Treatment of Tokenization in Psycholinguistics","created":"2024-10-03","cited_by_count":13,"area":null},{"id":"2505.21627","title":"Is Your LLM Overcharging You? Tokenization, Transparency, and Incentives","created":"2025-05-27","cited_by_count":13,"area":null},{"id":"2506.14123","title":"Sampling from Your Language Model One Byte at a Time","created":"2025-06-17","cited_by_count":7,"area":null},{"id":"2305.05461","title":"What is the best recipe for character-level encoder-only modelling?","created":"2023-05-09","cited_by_count":6,"area":null},{"id":"2311.00658","title":"Explicit Morphological Knowledge Improves Pre-training of Language Models for Hebrew","created":"2023-11-01","cited_by_count":5,"area":null},{"id":"2404.06508","title":"On the Effect of (Near) Duplicate Subwords in Language Modelling","created":"2024-04-09","cited_by_count":5,"area":null},{"id":"2402.17527","title":"Predict the Next Word: Humans exhibit uncertainty in this task and language models _____","created":"2024-02-27","cited_by_count":4,"area":null},{"id":"2510.05181","title":"Auditing Pay-Per-Token in Large Language Models","created":"2025-10-05","cited_by_count":4,"area":null},{"id":"2510.15346","title":"When to Ensemble: Identifying Token-Level Points for Stable and Fast LLM Ensembling","created":"2025-10-17","cited_by_count":3,"area":null},{"id":"2602.04289","title":"Proxy Compression for Language Modeling","created":"2026-02-04","cited_by_count":3,"area":null},{"id":"2607.26831","title":"Language Models are not Equally Robust to Non-Canonical Tokenization across Languages","created":"2026-07-29","cited_by_count":2,"area":null},{"id":"2307.16400","title":"SelfSeg: A Self-supervised Sub-word Segmentation Method for Neural Machine Translation","created":"2023-07-31","cited_by_count":1,"area":null},{"id":"2508.06621","title":"Train It and Forget It: Merge Lists are Unnecessary for BPE Inference in Language Models","created":"2025-08-08","cited_by_count":1,"area":null},{"id":"2506.06446","title":"Tokenization Multiplicity Leads to Arbitrary Price Variation in LLM-as-a-service","created":"2025-06-06","cited_by_count":0,"area":null},{"id":"2606.23566","title":"LangMAP: A Language-Adaptive Approach to Tokenization","created":"2026-06-22","cited_by_count":0,"area":null}],"next_cursor":null}