{"total":26,"rows":[{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2212.10769","title":"Uncontrolled Lexical Exposure Leads to Overestimation of Compositional Generalization in Pretrained Models","created":"2022-12-21","cited_by_count":26,"area":null},{"id":"2206.07615","title":"The SIGMORPHON 2022 Shared Task on Morpheme Segmentation","created":"2022-06-15","cited_by_count":25,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2204.14268","title":"How Robust is Neural Machine Translation to Language Imbalance in Multilingual Tokenizer Training?","created":"2022-04-29","cited_by_count":20,"area":null},{"id":"2312.11779","title":"Tokenization Matters: Navigating Data-Scarce Tokenization for Gender Inclusive Language Technologies","created":"2023-12-19","cited_by_count":16,"area":null},{"id":"2105.12410","title":"Joint Optimization of Tokenization and Downstream Model","created":"2021-05-26","cited_by_count":15,"area":null},{"id":"2301.13779","title":"FLAME: A small language model for spreadsheet formulas","created":"2023-01-31","cited_by_count":15,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2307.07262","title":"MorphPiece : A Linguistic Tokenizer for Large Language Models","created":"2023-07-14","cited_by_count":9,"area":null},{"id":"2502.07057","title":"Tokenization Standards for Linguistic Integrity: Turkish as a Benchmark","created":"2025-02-10","cited_by_count":8,"area":null},{"id":"2305.14214","title":"CompoundPiece: Evaluating and Improving Decompounding Performance of Language Models","created":"2023-05-23","cited_by_count":7,"area":null},{"id":"2309.12491","title":"Exploring the Impact of Training Data Distribution and Subword Tokenization on Gender Bias in Machine Translation","created":"2023-09-21","cited_by_count":7,"area":null},{"id":"2209.13279","title":"Improving Multilingual Neural Machine Translation System for Indic Languages","created":"2022-09-27","cited_by_count":6,"area":null},{"id":"2305.00606","title":"Low-Resourced Machine Translation for Senegalese Wolof Language","created":"2023-05-01","cited_by_count":5,"area":null},{"id":"2305.08518","title":"Beqi: Revitalize the Senegalese Wolof Language with a Robust Spelling Corrector","created":"2023-05-15","cited_by_count":4,"area":null},{"id":"2306.07764","title":"Tokenization with Factorized Subword Encoding","created":"2023-06-13","cited_by_count":4,"area":null},{"id":"2508.13058","title":"Do\\u{g}al Dil \\.I\\c{s}lemede Tokenizasyon Standartlar{\\i} ve \\\"Ol\\c{c}\\\"um\\\"u: T\\\"urk\\c{c}e \\\"Uzerinden B\\\"uy\\\"uk Dil Modellerinin Kar\\c{s}{\\i}la\\c{s}t{\\i}rmal{\\i} Analizi","created":"2025-08-18","cited_by_count":4,"area":null},{"id":"2208.05225","title":"How Effective is Byte Pair Encoding for Out-Of-Vocabulary Words in Neural Machine Translation?","created":"2022-08-10","cited_by_count":3,"area":null},{"id":"2404.02421","title":"Revisiting subword tokenization: A case study on affixal negation in large language models","created":"2024-04-03","cited_by_count":2,"area":null},{"id":"2602.05648","title":"Modelling the Morphology of Verbal Paradigms: A Case Study in the Tokenization of Turkish and Hebrew","created":"2026-02-05","cited_by_count":2,"area":null},{"id":"2606.20993","title":"Phonemes to the Rescue: Multilingual Tokenization Based on International Phonetic Alphabet","created":"2026-06-18","cited_by_count":2,"area":null},{"id":"2403.10963","title":"Pointer-Generator Networks for Low-Resource Machine Translation: Don't Copy That!","created":"2024-03-16","cited_by_count":1,"area":null},{"id":"2410.23656","title":"Morphological Typology in BPE Subword Productivity and Language Modeling","created":"2024-10-31","cited_by_count":1,"area":null},{"id":"2406.15245","title":"Unsupervised Morphological Tree Tokenizer","created":"2024-06-21","cited_by_count":0,"area":null},{"id":"2506.06446","title":"Tokenization Multiplicity Leads to Arbitrary Price Variation in LLM-as-a-service","created":"2025-06-06","cited_by_count":0,"area":null}],"next_cursor":null}