{"total":89,"rows":[{"id":"2507.07955","title":"Dynamic Chunking for End-to-End Hierarchical Sequence Modeling","created":"2025-07-10","cited_by_count":72,"area":null},{"id":"2503.13423","title":"SuperBPE: Space Travel for Language Models","created":"2025-03-17","cited_by_count":63,"area":null},{"id":"2403.06265","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","created":"2024-03-10","cited_by_count":61,"area":null},{"id":"2410.05864","title":"From Tokens to Words: On the Inner Lexicon of LLMs","created":"2024-10-08","cited_by_count":48,"area":null},{"id":"2403.01289","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","created":"2024-03-02","cited_by_count":34,"area":null},{"id":"2504.00178","title":"Boundless Byte Pair Encoding: Breaking the Pre-tokenization Barrier","created":"2025-03-31","cited_by_count":32,"area":null},{"id":"2507.06378","title":"Evaluating Morphological Alignment of Tokenizers in 70 Languages","created":"2025-07-08","cited_by_count":22,"area":null},{"id":"2502.18795","title":"Anything Goes? A Crosslinguistic Study of (Im)possible Language Learning in LMs","created":"2025-02-26","cited_by_count":20,"area":null},{"id":"2406.16829","title":"Understanding and Mitigating Tokenization Bias in Language Models","created":"2024-06-24","cited_by_count":18,"area":null},{"id":"2502.13581","title":"ActionPiece: Contextually Tokenizing Action Sequences for Generative Recommendation","created":"2025-02-19","cited_by_count":18,"area":null},{"id":"2404.03626","title":"Training LLMs over Neurally Compressed Text","created":"2024-04-04","cited_by_count":17,"area":null},{"id":"2502.16060","title":"Tokenizing Single-Channel EEG with Time-Frequency Motif Learning","created":"2025-02-22","cited_by_count":17,"area":null},{"id":"2404.17808","title":"Scaffold-BPE: Enhancing Byte Pair Encoding for Large Language Models with Simple and Effective Scaffold Token Removal","created":"2024-04-27","cited_by_count":16,"area":null},{"id":"2411.08671","title":"Theoretical Analysis of Byte-Pair Encoding","created":"2024-11-13","cited_by_count":16,"area":null},{"id":"2412.16326","title":"When Worse is Better: Navigating the compression-generation tradeoff in visual tokenization","created":"2024-12-20","cited_by_count":15,"area":null},{"id":"2502.20273","title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","created":"2025-02-27","cited_by_count":15,"area":null},{"id":"2503.19693","title":"AdaptiVocab: Enhancing LLM Efficiency in Focused Domains through Lightweight Vocabulary Adaptation","created":"2025-03-25","cited_by_count":15,"area":null},{"id":"2511.01380","title":"Confounding Factors in Relating Model Performance to Morphology","created":"2025-11-03","cited_by_count":14,"area":null},{"id":"2406.13560","title":"Lexically Grounded Subword Segmentation","created":"2024-06-19","cited_by_count":13,"area":null},{"id":"2411.07854","title":"Tucano: Advancing Neural Text Generation for Portuguese","created":"2024-11-12","cited_by_count":13,"area":null},{"id":"2502.15343","title":"Tokenization is Sensitive to Language Variation","created":"2025-02-21","cited_by_count":13,"area":null},{"id":"2505.24689","title":"BPE Stays on SCRIPT: Structured Encoding for Robust Multilingual Pretokenization","created":"2025-05-30","cited_by_count":12,"area":null},{"id":"2506.10766","title":"One Tokenizer To Rule Them All: Emergent Language Plasticity via Multilingual Tokenizers","created":"2025-06-12","cited_by_count":12,"area":null},{"id":"2506.03101","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","created":"2025-06-03","cited_by_count":11,"area":null},{"id":"2601.07220","title":"The Roots of Performance Disparity in Multilingual Language Models: Intrinsic Modeling Difficulty or Design Choices?","created":"2026-01-12","cited_by_count":11,"area":null},{"id":"2505.03798","title":"Position: Foundation Models Need Digital Twin Representations","created":"2025-05-01","cited_by_count":10,"area":null},{"id":"2502.07057","title":"Tokenization Standards for Linguistic Integrity: Turkish as a Benchmark","created":"2025-02-10","cited_by_count":8,"area":null},{"id":"2508.15390","title":"Exploiting Vocabulary Frequency Imbalance in Language Model Pre-training","created":"2025-08-21","cited_by_count":7,"area":null},{"id":"2510.21909","title":"Explaining and Mitigating Crosslingual Tokenizer Inequities","created":"2025-10-24","cited_by_count":7,"area":null},{"id":"2502.12835","title":"Subword models struggle with word learning, but surprisal hides it","created":"2025-02-18","cited_by_count":6,"area":null},{"id":"2503.14433","title":"Splintering Nonconcatenative Languages for Better Tokenization","created":"2025-03-18","cited_by_count":6,"area":null},{"id":"2505.16088","title":"Date Fragments: A Hidden Bottleneck of Tokenization for Temporal Reasoning","created":"2025-05-22","cited_by_count":6,"area":"lr"},{"id":"2601.13260","title":"Stop Taking Tokenizers for Granted: They Are Core Design Decisions in Large Language Models","created":"2026-01-19","cited_by_count":6,"area":null},{"id":"2603.15644","title":"Tokenization Tradeoffs in Structured EHR Foundation Models","created":"2026-03-03","cited_by_count":6,"area":null},{"id":"2501.06246","title":"A Partition Cover Approach to Tokenization","created":"2025-01-08","cited_by_count":5,"area":null},{"id":"2606.27019","title":"MinGram: A Minimalist Unigram Tokenizer with High Compression and Competitive Morphological Alignment","created":"2026-06-25","cited_by_count":5,"area":null},{"id":"2508.13058","title":"Do\\u{g}al Dil \\.I\\c{s}lemede Tokenizasyon Standartlar{\\i} ve \\\"Ol\\c{c}\\\"um\\\"u: T\\\"urk\\c{c}e \\\"Uzerinden B\\\"uy\\\"uk Dil Modellerinin Kar\\c{s}{\\i}la\\c{s}t{\\i}rmal{\\i} Analizi","created":"2025-08-18","cited_by_count":4,"area":null},{"id":"2510.21805","title":"DiffGRM: Diffusion-based Generative Recommendation Model","created":"2025-10-21","cited_by_count":4,"area":"diffusion"},{"id":"2512.12641","title":"Which Pieces Does Unigram Tokenization Really Need?","created":"2025-12-14","cited_by_count":4,"area":null},{"id":"2512.20757","title":"TokSuite: Measuring the Impact of Tokenizer Choice on Language Model Behavior","created":"2025-12-23","cited_by_count":4,"area":null},{"id":"2605.01188","title":"Compute Optimal Tokenization","created":"2026-05-02","cited_by_count":4,"area":null},{"id":"2503.15564","title":"GReaTER: Generate Realistic Tabular data after data Enhancement and Reduction","created":"2025-03-19","cited_by_count":3,"area":null},{"id":"2505.09068","title":"S-DAT: A Multilingual, GenAI-Driven Framework for Automated Divergent Thinking Assessment","created":"2025-05-14","cited_by_count":3,"area":"lr"},{"id":"2511.05578","title":"UTF-8 Plumbing: Byte-level Tokenizers Unavoidably Enable LLMs to Generate Ill-formed UTF-8","created":"2025-11-05","cited_by_count":3,"area":null},{"id":"2602.04289","title":"Proxy Compression for Language Modeling","created":"2026-02-04","cited_by_count":3,"area":null},{"id":"2608.25089","title":"Apples to Apples? Towards Comparable Crosslingual Language Model Evaluation","created":"2026-08-25","cited_by_count":3,"area":null},{"id":"2412.13377","title":"DateLogicQA: Benchmarking Temporal Biases in Large Language Models","created":"2024-12-17","cited_by_count":2,"area":null},{"id":"2412.15250","title":"An Enhanced Text Compression Approach Using Transformer-based Language Models","created":"2024-12-15","cited_by_count":2,"area":null},{"id":"2510.07613","title":"Vocabulary embeddings organize linguistic structure early in language model training","created":"2025-10-08","cited_by_count":2,"area":null},{"id":"2603.10961","title":"Bio-Inspired Self-Supervised Learning for Wrist-worn Accelerometer Data","created":"2026-03-11","cited_by_count":2,"area":null},{"id":"2604.27263","title":"Decoupling the Benefits of Subword Tokenization for Language Model Training via Byte-level Simulation","created":"2026-04-29","cited_by_count":2,"area":null},{"id":"2605.22705","title":"Tokenization with Split Trees","created":"2026-05-21","cited_by_count":2,"area":null},{"id":"2608.18062","title":"TokEval: A Tokenizer Evaluation Suite","created":"2026-08-18","cited_by_count":2,"area":null},{"id":"2505.21680","title":"multivariateGPT: a decoder-only transformer for multivariate categorical and numeric data","created":"2025-05-27","cited_by_count":1,"area":null},{"id":"2511.09709","title":"Contextual morphologically-guided tokenization for Latin encoder models","created":"2025-11-12","cited_by_count":1,"area":null},{"id":"2511.20849","title":"Length-MAX Tokenizer for Language Models","created":"2025-11-25","cited_by_count":1,"area":null},{"id":"2512.21913","title":"GQ-VAE: A gated quantized VAE for learning variable length tokens","created":"2025-12-26","cited_by_count":1,"area":null},{"id":"2601.18536","title":"Evaluating Morphological Plausibility of Subword Tokenization via Statistical Alignment with Morpho-Syntactic Features","created":"2026-01-26","cited_by_count":1,"area":null},{"id":"2602.10657","title":"Benchmarks Are Not That Out of Distribution: Word Overlap Predicts Performance","created":"2026-02-11","cited_by_count":1,"area":null},{"id":"2603.03583","title":"ByteFlow: Language Modeling through Adaptive Byte Compression without a Tokenizer","created":"2026-03-03","cited_by_count":1,"area":null},{"id":"2605.13485","title":"Effective Context in Transformers: An Analysis of Fragmentation and Tokenization","created":"2026-05-13","cited_by_count":1,"area":null},{"id":"2607.23362","title":"Joint Optimization for Greedy Longest-match Tokenization","created":"2026-07-25","cited_by_count":1,"area":null},{"id":"2608.00837","title":"Pruned BPE: Post-training Visibility Pruning and Token Reallocation for Byte Pair Encoding","created":"2026-08-01","cited_by_count":1,"area":null},{"id":"2507.07824","title":"Conditional Unigram Tokenization with Parallel Data","created":"2025-07-10","cited_by_count":0,"area":null},{"id":"2508.01685","title":"Innovative tokenisation of structured data for LLM training","created":"2025-08-03","cited_by_count":0,"area":null},{"id":"2509.08812","title":"MoVoC: Morphology-Aware Subword Construction for Geez Script Languages","created":"2025-09-10","cited_by_count":0,"area":null},{"id":"2509.11425","title":"FuseCodec: Semantic-Contextual Fusion and Supervision for Neural Codecs","created":"2025-09-14","cited_by_count":0,"area":null},{"id":"2512.07522","title":"LIME: Making LLM Data More Efficient with Linguistic Metadata Embeddings","created":"2025-12-08","cited_by_count":0,"area":null},{"id":"2601.05833","title":"Peek2: Regex-free Byte-level Byte-Pair Encoding Pretokenizer for LLM Inference on Edge Devices","created":"2026-01-09","cited_by_count":0,"area":null},{"id":"2601.11518","title":"How Long Is a Piece of String? A Brief Empirical Analysis of Tokenizers","created":"2026-01-16","cited_by_count":0,"area":null},{"id":"2602.13958","title":"Chemical Language Models for Natural Products: A State-Space Model Approach","created":"2026-02-15","cited_by_count":0,"area":null},{"id":"2604.08698","title":"EvoLen: Evolution-Guided Tokenization for DNA Language Model","created":"2026-04-09","cited_by_count":0,"area":null},{"id":"2605.12106","title":"Large Language Models as Amortized Pareto-Front Generators for Constrained Bi-Objective Convex Optimization","created":"2026-05-12","cited_by_count":0,"area":null},{"id":"2606.08562","title":"Inside the LLM Word Factory","created":"2026-06-07","cited_by_count":0,"area":null},{"id":"2606.15521","title":"Emergent retokenization symmetry in large language models: phenomenology and applications","created":"2026-06-14","cited_by_count":0,"area":null},{"id":"2606.19626","title":"Toten: A Knowledge-Based System For Structure-Preserving Representation Of Physical Quantities And Technical Notation In Brazilian Portuguese","created":"2026-06-17","cited_by_count":0,"area":null},{"id":"2606.23566","title":"LangMAP: A Language-Adaptive Approach to Tokenization","created":"2026-06-22","cited_by_count":0,"area":null},{"id":"2607.05691","title":"Where to cut, how deep: BPE and Unigram-LM on chemistry SMILES","created":"2026-07-06","cited_by_count":0,"area":null},{"id":"2608.17325","title":"What Tokens are Learned when Tokenization is Optimized Jointly with Language Modeling?","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2608.18025","title":"How Far Should Tokenization Go? Predictive Effectiveness and Relational Losslessness","created":"2026-08-18","cited_by_count":0,"area":null},{"id":"2609.00378","title":"Removable and Irreducible: A Token-Cost Ledger for the Multilingual Tokenization Tax","created":"2026-07-17","cited_by_count":0,"area":null},{"id":"2609.01386","title":"When Tokenization is Secretly Output Supervision","created":"2026-09-01","cited_by_count":0,"area":null},{"id":"2609.05022","title":"MoirfEolas and Cr\\'iochScore: Developing Resources for and the Evaluation of Tokenization Alignment with Irish Morphology","created":"2026-09-04","cited_by_count":0,"area":null},{"id":"2609.06898","title":"Dynamic-Programming-Guided Hierarchical BPE and Empirical Analysis of Vocabulary Pruning","created":"2026-09-07","cited_by_count":0,"area":null},{"id":"2609.15528","title":"To Each Language Its Tokenizer: Modular Tokenizers for Efficient Multilingual LLMs","created":"2026-09-14","cited_by_count":0,"area":null},{"id":"2609.18720","title":"LocQE: Principled Domain Adaptation for Localisation Quality Estimation by Leveraging Post-Edits","created":"2026-09-16","cited_by_count":0,"area":null},{"id":"2609.35869","title":"The Price of Token Boundaries: Compression Certificates and Prediction","created":"2026-09-26","cited_by_count":0,"area":null},{"id":"2610.01127","title":"Counting and Min-Cost Encoding for Tokenization in Large Language Models","created":"2026-10-01","cited_by_count":0,"area":null},{"id":"2610.05597","title":"More Than Words: Compositional Tokenization for Efficient Language Models","created":"2026-10-04","cited_by_count":0,"area":null}],"next_cursor":null}