{"total":19,"rows":[{"id":"2402.18376","title":"Tokenization Is More Than Compression","created":"2024-02-28","cited_by_count":89,"area":null},{"id":"2403.06265","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","created":"2024-03-10","cited_by_count":61,"area":null},{"id":"2207.06991","title":"Language Modelling with Pixels","created":"2022-07-14","cited_by_count":55,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2206.07615","title":"The SIGMORPHON 2022 Shared Task on Morpheme Segmentation","created":"2022-06-15","cited_by_count":25,"area":null},{"id":"2407.11606","title":"The Foundations of Tokenization: Statistical and Computational Concerns","created":"2024-07-16","cited_by_count":22,"area":null},{"id":"2306.09572","title":"How do different tokenizers perform on downstream tasks in scriptio continua languages?: A case study in Japanese","created":"2023-06-16","cited_by_count":20,"area":null},{"id":"2402.10712","title":"An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Language Model Inference","created":"2024-02-16","cited_by_count":18,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2404.13292","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","created":"2024-04-20","cited_by_count":8,"area":null},{"id":"2404.00397","title":"An Analysis of BPE Vocabulary Trimming in Neural Machine Translation","created":"2024-03-30","cited_by_count":7,"area":null},{"id":"2406.15175","title":"Enhancing Idiomatic Representation in Multiple Languages via an Adaptive Contrastive Triplet Loss","created":"2024-06-21","cited_by_count":7,"area":null},{"id":"2308.13315","title":"Construction Grammar and Language Models","created":"2023-08-25","cited_by_count":2,"area":null},{"id":"2401.07923","title":"Word Boundary Information Isn't Useful for Encoder Language Models","created":"2024-01-15","cited_by_count":2,"area":null},{"id":"2411.02610","title":"Investigating Idiomaticity in Word Representations","created":"2024-11-04","cited_by_count":1,"area":null},{"id":"2506.18639","title":"ByteSpan: Information-Driven Subword Tokenisation","created":"2025-06-23","cited_by_count":1,"area":null},{"id":"2512.13552","title":"PrahokBART: A Pre-trained Sequence-to-Sequence Model for Khmer Natural Language Generation","created":"2025-12-15","cited_by_count":1,"area":null},{"id":"2608.08847","title":"Explicit Boundary Markers for Subword Vocabularies","created":"2026-08-09","cited_by_count":0,"area":null}],"next_cursor":null}