{"total":50,"rows":[{"id":"2103.06874","title":"CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation","created":"2021-03-11","cited_by_count":232,"area":null},{"id":"2305.15425","title":"Language Model Tokenizers Introduce Unfairness Between Languages","created":"2023-05-17","cited_by_count":214,"area":null},{"id":"2305.13707","title":"Do All Languages Cost the Same? Tokenization in the Era of Commercial Language Models","created":"2023-05-23","cited_by_count":192,"area":null},{"id":"2103.10730","title":"MuRIL: Multilingual Representations for Indian Languages","created":"2021-03-19","cited_by_count":179,"area":null},{"id":"2304.09151","title":"UniMax: Fairer and more Effective Language Sampling for Large-Scale Multilingual Pretraining","created":"2023-04-18","cited_by_count":151,"area":null},{"id":"2205.06266","title":"Lifting the Curse of Multilinguality by Pre-training Modular Transformers","created":"2022-05-12","cited_by_count":146,"area":null},{"id":"2112.10508","title":"Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP","created":"2021-12-20","cited_by_count":132,"area":null},{"id":"2407.13623","title":"Scaling Laws with Vocabulary: Larger Models Deserve Larger Vocabularies","created":"2024-07-18","cited_by_count":118,"area":null},{"id":"2305.12182","title":"Glot500: Scaling Multilingual Corpora and Language Models to 500 Languages","created":"2023-05-20","cited_by_count":116,"area":null},{"id":"2301.10472","title":"XLM-V: Overcoming the Vocabulary Bottleneck in Multilingual Masked Language Models","created":"2023-01-25","cited_by_count":88,"area":null},{"id":"2105.04371","title":"Poolingformer: Long Document Modeling with Pooling Attention","created":"2021-05-10","cited_by_count":60,"area":null},{"id":"2207.06991","title":"Language Modelling with Pixels","created":"2022-07-14","cited_by_count":55,"area":null},{"id":"2107.00676","title":"A Primer on Pretrained Multilingual Language Models","created":"2021-07-01","cited_by_count":49,"area":null},{"id":"2103.08490","title":"Multi-view Subword Regularization","created":"2021-03-15","cited_by_count":40,"area":null},{"id":"2403.10691","title":"MYTE: Morphology-Driven Byte Encoding for Better and Fairer Multilingual Language Modeling","created":"2024-03-15","cited_by_count":40,"area":null},{"id":"2407.08818","title":"MAGNET: Improving the Multilingual Fairness of Language Models with Adaptive Gradient-Based Tokenization","created":"2024-07-11","cited_by_count":31,"area":null},{"id":"2310.13348","title":"Analyzing Cognitive Plausibility of Subword Tokenization","created":"2023-10-20","cited_by_count":30,"area":null},{"id":"2203.01976","title":"Overlap-based Vocabulary Generation Improves Cross-lingual Transfer Among Related Languages","created":"2022-03-03","cited_by_count":25,"area":null},{"id":"2109.07306","title":"Allocating Large Vocabulary Capacity for Cross-lingual Language Model Pre-training","created":"2021-09-15","cited_by_count":22,"area":null},{"id":"2305.14280","title":"Multilingual Pixel Representations for Translation and Effective Cross-lingual Transfer","created":"2023-05-23","cited_by_count":22,"area":null},{"id":"2106.09063","title":"Specializing Multilingual Language Models: An Empirical Study","created":"2021-06-16","cited_by_count":21,"area":null},{"id":"2206.07043","title":"Text Generation with Text-Editing Models","created":"2022-06-14","cited_by_count":21,"area":null},{"id":"2204.14268","title":"How Robust is Neural Machine Translation to Language Imbalance in Multilingual Tokenizer Training?","created":"2022-04-29","cited_by_count":20,"area":null},{"id":"2305.15020","title":"An Efficient Multilingual Language Model Compression through Vocabulary Trimming","created":"2023-05-24","cited_by_count":20,"area":null},{"id":"2508.04796","title":"Parity-Aware Byte-Pair Encoding: Improving Cross-lingual Fairness in Tokenization","created":"2025-08-06","cited_by_count":20,"area":null},{"id":"2305.11550","title":"Viewing Knowledge Transfer in Multilingual Machine Translation Through a Representational Lens","created":"2023-05-19","cited_by_count":15,"area":null},{"id":"2208.04826","title":"Compositional Evaluation on Japanese Textual Entailment and Similarity","created":"2022-08-09","cited_by_count":14,"area":null},{"id":"2106.02834","title":"MergeDistill: Merging Pre-trained Language Models using Distillation","created":"2021-06-05","cited_by_count":13,"area":null},{"id":"2204.10815","title":"A Vocabulary-Free Multilingual Neural Tokenizer for End-to-End Task Learning","created":"2022-04-22","cited_by_count":12,"area":null},{"id":"2311.09071","title":"How Vocabulary Sharing Facilitates Multilingualism in LLaMA?","created":"2023-11-15","cited_by_count":12,"area":null},{"id":"2210.07095","title":"Incorporating Context into Subword Vocabularies","created":"2022-10-13","cited_by_count":11,"area":null},{"id":"2509.18750","title":"False Friends Are Not Foes: Investigating Vocabulary Overlap in Multilingual Language Models","created":"2025-09-23","cited_by_count":11,"area":null},{"id":"2308.00683","title":"CodeBPE: Investigating Subtokenization Options for Large Language Model Pretraining on Source Code","created":"2023-08-01","cited_by_count":9,"area":null},{"id":"2410.09644","title":"Adapters for Altering LLM Vocabularies: What Languages Benefit the Most?","created":"2024-10-12","cited_by_count":9,"area":null},{"id":"2109.01100","title":"How Suitable Are Subword Segmentation Strategies for Translating Non-Concatenative Morphology?","created":"2021-09-02","cited_by_count":7,"area":null},{"id":"2406.17535","title":"Disce aut Deficere: Evaluating LLMs Proficiency on the INVALSI Italian Benchmark","created":"2024-06-25","cited_by_count":5,"area":null},{"id":"2410.05021","title":"DEPT: Decoupled Embeddings for Pre-training Language Models","created":"2024-10-07","cited_by_count":5,"area":null},{"id":"2507.03003","title":"Breaking Physical and Linguistic Borders: Multilingual Federated Prompt Tuning for Low-Resource Languages","created":"2025-07-02","cited_by_count":5,"area":null},{"id":"2109.00271","title":"Discovering Representation Sprachbund For Multilingual Pre-Training","created":"2021-09-01","cited_by_count":4,"area":null},{"id":"2403.20157","title":"A Systematic Analysis of Subwords and Cross-Lingual Transfer in Multilingual Translation","created":"2024-03-29","cited_by_count":3,"area":null},{"id":"2410.17532","title":"Responsible Multilingual Large Language Models: A Survey of Development, Applications, and Societal Impact","created":"2024-10-23","cited_by_count":3,"area":null},{"id":"2501.09127","title":"Multilingual LLMs Struggle to Link Orthography and Semantics in Bilingual Word Processing","created":"2025-01-15","cited_by_count":3,"area":null},{"id":"2208.01912","title":"Cross-Lingual Knowledge Transfer for Clinical Phenotyping","created":"2022-08-03","cited_by_count":2,"area":null},{"id":"2404.12460","title":"NLP-enabled Trajectory Map-matching in Urban Road Networks using a Transformer-based Encoder-decoder","created":"2024-04-18","cited_by_count":2,"area":null},{"id":"2506.17789","title":"Multilingual Tokenization through the Lens of Indian Languages: Challenges and Insights","created":"2025-06-21","cited_by_count":2,"area":null},{"id":"2507.16323","title":"SpeLLM: Character-Level Multi-Head Decoding","created":"2025-07-22","cited_by_count":2,"area":null},{"id":"2205.08497","title":"Feature Aggregation in Zero-Shot Cross-Lingual Transfer Using Multilingual BERT","created":"2022-05-17","cited_by_count":1,"area":null},{"id":"2403.10963","title":"Pointer-Generator Networks for Low-Resource Machine Translation: Don't Copy That!","created":"2024-03-16","cited_by_count":1,"area":null},{"id":"2605.04196","title":"The Impact of Vocabulary Overlaps on Knowledge Transfer in Multilingual Machine Translation","created":"2026-05-05","cited_by_count":0,"area":null},{"id":"2609.15528","title":"To Each Language Its Tokenizer: Modular Tokenizers for Efficient Multilingual LLMs","created":"2026-09-14","cited_by_count":0,"area":null}],"next_cursor":null}