{"total":38,"rows":[{"id":"2106.12672","title":"Charformer: Fast Character Transformers via Gradient-based Subword Tokenization","created":"2021-06-23","cited_by_count":160,"area":null},{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2109.00486","title":"Survey of Low-Resource Machine Translation","created":"2021-09-01","cited_by_count":92,"area":null},{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2103.08490","title":"Multi-view Subword Regularization","created":"2021-03-15","cited_by_count":40,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2109.02550","title":"You should evaluate your language model on marginal likelihood over tokenisations","created":"2021-09-06","cited_by_count":30,"area":null},{"id":"2206.07615","title":"The SIGMORPHON 2022 Shared Task on Morpheme Segmentation","created":"2022-06-15","cited_by_count":25,"area":null},{"id":"2204.04058","title":"Improving Tokenisation by Alternative Treatment of Spaces","created":"2022-04-08","cited_by_count":19,"area":null},{"id":"2306.17757","title":"Should you marginalize over possible tokenizations?","created":"2023-06-30","cited_by_count":18,"area":null},{"id":"2502.13581","title":"ActionPiece: Contextually Tokenizing Action Sequences for Generative Recommendation","created":"2025-02-19","cited_by_count":18,"area":null},{"id":"2404.17808","title":"Scaffold-BPE: Enhancing Byte Pair Encoding for Large Language Models with Simple and Effective Scaffold Token Removal","created":"2024-04-27","cited_by_count":16,"area":null},{"id":"2105.12410","title":"Joint Optimization of Tokenization and Downstream Model","created":"2021-05-26","cited_by_count":15,"area":null},{"id":"2402.14614","title":"Two Counterexamples to Tokenization and the Noiseless Channel","created":"2024-02-22","cited_by_count":13,"area":null},{"id":"2406.13560","title":"Lexically Grounded Subword Segmentation","created":"2024-06-19","cited_by_count":13,"area":null},{"id":"2105.13782","title":"How to Split: the Effect of Word Segmentation on Gender Bias in Speech Translation","created":"2021-05-28","cited_by_count":11,"area":null},{"id":"2210.07095","title":"Incorporating Context into Subword Vocabularies","created":"2022-10-13","cited_by_count":11,"area":null},{"id":"2305.03029","title":"What changes when you randomly choose BPE merge operations? Not much","created":"2023-05-04","cited_by_count":10,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2212.09912","title":"Tokenization Consistency Matters for Generative Models on Extractive NLP Tasks","created":"2022-12-19","cited_by_count":7,"area":null},{"id":"2305.07005","title":"Subword Segmental Machine Translation: Unifying Segmentation and Target Sentence Generation","created":"2023-05-11","cited_by_count":7,"area":null},{"id":"2205.11490","title":"Local Byte Fusion for Neural Machine Translation","created":"2022-05-23","cited_by_count":5,"area":null},{"id":"2208.05225","title":"How Effective is Byte Pair Encoding for Out-Of-Vocabulary Words in Neural Machine Translation?","created":"2022-08-10","cited_by_count":3,"area":null},{"id":"2211.15965","title":"Extending the Subwording Model of Multilingual Pretrained Models for New Languages","created":"2022-11-29","cited_by_count":3,"area":null},{"id":"2309.04459","title":"Subwords as Skills: Tokenization for Sparse-Reward Reinforcement Learning","created":"2023-09-08","cited_by_count":3,"area":null},{"id":"2403.07567","title":"Triples-to-isiXhosa (T2X): Addressing the Challenges of Low-Resource Agglutinative Data-to-Text Generation","created":"2024-03-12","cited_by_count":3,"area":null},{"id":"2103.07186","title":"Dynamic Acoustic Unit Augmentation With BPE-Dropout for Low-Resource End-to-End Speech Recognition","created":"2021-03-12","cited_by_count":2,"area":null},{"id":"2511.09197","title":"The Learning Dynamics of Subword Segmentation for Morphologically Diverse Languages","created":"2025-11-12","cited_by_count":2,"area":null},{"id":"2306.02383","title":"Evolution of Efficient Symbolic Communication Codes","created":"2023-06-04","cited_by_count":1,"area":null},{"id":"2307.16400","title":"SelfSeg: A Self-supervised Sub-word Segmentation Method for Neural Machine Translation","created":"2023-07-31","cited_by_count":1,"area":null},{"id":"2512.13552","title":"PrahokBART: A Pre-trained Sequence-to-Sequence Model for Khmer Natural Language Generation","created":"2025-12-15","cited_by_count":1,"area":null},{"id":"2605.13436","title":"Pretraining Language Models with Subword Regularization: An Empirical Study of BPE Dropout in Low-Resource NLP","created":"2026-05-13","cited_by_count":1,"area":null},{"id":"2202.13590","title":"LCP-dropout: Compression-based Multiple Subword Segmentation for Neural Machine Translation","created":"2022-02-28","cited_by_count":0,"area":null},{"id":"2302.13475","title":"Elementwise Language Representation","created":"2023-02-27","cited_by_count":0,"area":null},{"id":"2304.10808","title":"Downstream Task-Oriented Neural Tokenizer Optimization with Vocabulary Restriction as Post Processing","created":"2023-04-21","cited_by_count":0,"area":null},{"id":"2304.10813","title":"Tokenization Preference for Human and Machine Learning Model: An Annotation Study","created":"2023-04-21","cited_by_count":0,"area":null},{"id":"2609.01386","title":"When Tokenization is Secretly Output Supervision","created":"2026-09-01","cited_by_count":0,"area":null},{"id":"2609.06898","title":"Dynamic-Programming-Guided Hierarchical BPE and Empirical Analysis of Vocabulary Pruning","created":"2026-09-07","cited_by_count":0,"area":null}],"next_cursor":null}