{"total":76,"rows":[{"id":"2307.03172","title":"Lost in the Middle: How Language Models Use Long Contexts","created":"2023-07-06","cited_by_count":3455,"area":null},{"id":"2104.00298","title":"EfficientNetV2: Smaller Models and Faster Training","created":"2021-04-01","cited_by_count":1135,"area":null},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103,"area":null},{"id":"2304.05969","title":"Localizing Model Behavior with Path Patching","created":"2023-04-12","cited_by_count":195,"area":null},{"id":"2306.15794","title":"HyenaDNA: Long-Range Genomic Sequence Modeling at Single Nucleotide Resolution","created":"2023-06-27","cited_by_count":194,"area":null},{"id":"2405.14782","title":"Lessons from the Trenches on Reproducible Evaluation of Language Models","created":"2024-05-23","cited_by_count":179,"area":null},{"id":"2112.10684","title":"Efficient Large Scale Language Modeling with Mixtures of Experts","created":"2021-12-20","cited_by_count":175,"area":"moe"},{"id":"2201.03533","title":"SCROLLS: Standardized CompaRison Over Long Language Sequences","created":"2022-01-10","cited_by_count":170,"area":null},{"id":"2305.07185","title":"MEGABYTE: Predicting Million-byte Sequences with Multiscale Transformers","created":"2023-05-12","cited_by_count":170,"area":null},{"id":"2303.06296","title":"Stabilizing Transformer Training by Preventing Attention Entropy Collapse","created":"2023-03-11","cited_by_count":161,"area":null},{"id":"2104.08704","title":"A Token-level Reference-free Hallucination Detection Benchmark for Free-form Text Generation","created":"2021-04-18","cited_by_count":138,"area":null},{"id":"2102.11090","title":"Position Information in Transformers: An Overview","created":"2021-02-22","cited_by_count":103,"area":null},{"id":"2103.13076","title":"Finetuning Pretrained Transformers into RNNs","created":"2021-03-24","cited_by_count":96,"area":null},{"id":"2109.09115","title":"Do Long-Range Language Models Actually Use Long-Range Context?","created":"2021-09-19","cited_by_count":92,"area":null},{"id":"2504.08165","title":"Findings of the BabyLM Challenge: Sample-Efficient Pretraining on Developmentally Plausible Corpora","created":"2025-04-10","cited_by_count":88,"area":null},{"id":"2110.09456","title":"NormFormer: Improved Transformer Pretraining with Extra Normalization","created":"2021-10-18","cited_by_count":83,"area":null},{"id":"2209.00099","title":"Efficient Methods for Natural Language Processing: A Survey","created":"2022-08-31","cited_by_count":78,"area":null},{"id":"2202.07765","title":"General-purpose, long-context autoregressive modeling with Perceiver AR","created":"2022-02-15","cited_by_count":60,"area":null},{"id":"2401.13660","title":"MambaByte: Token-free Selective State Space Model","created":"2024-01-24","cited_by_count":59,"area":"ssm"},{"id":"2404.09937","title":"Compression Represents Intelligence Linearly","created":"2024-04-15","cited_by_count":56,"area":null},{"id":"2312.10523","title":"Paloma: A Benchmark for Evaluating Language Model Fit","created":"2023-12-16","cited_by_count":54,"area":null},{"id":"2308.12108","title":"The Local Learning Coefficient: A Singularity-Aware Complexity Measure","created":"2023-08-23","cited_by_count":51,"area":null},{"id":"2203.06211","title":"Staged Training for Transformer Language Models","created":"2022-03-11","cited_by_count":49,"area":null},{"id":"2402.02364","title":"Loss Landscape Degeneracy and Stagewise Development in Transformers","created":"2024-02-04","cited_by_count":40,"area":null},{"id":"2108.06084","title":"The Stability-Efficiency Dilemma: Investigating Sequence Length Warmup for Training GPT Models","created":"2021-08-13","cited_by_count":36,"area":null},{"id":"2306.08620","title":"Anticipatory Music Transformer","created":"2023-06-14","cited_by_count":36,"area":null},{"id":"2102.12459","title":"When Attention Meets Fast Recurrence: Training Language Models with Reduced Compute","created":"2021-02-24","cited_by_count":34,"area":null},{"id":"2404.14408","title":"SpaceByte: Towards Deleting Tokenization from Large Language Modeling","created":"2024-04-22","cited_by_count":34,"area":null},{"id":"2401.07872","title":"The What, Why, and How of Context Length Extension Techniques in Large Language Models -- A Detailed Survey","created":"2024-01-15","cited_by_count":33,"area":null},{"id":"2110.08743","title":"GNN-LM: Language Modeling based on Global Contexts via GNN","created":"2021-10-17","cited_by_count":27,"area":null},{"id":"2304.03589","title":"On Efficient Training of Large-Scale Deep Learning Models: A Literature Review","created":"2023-04-07","cited_by_count":26,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2112.07210","title":"Simple Local Attentions Remain Competitive for Long-Context Tasks","created":"2021-12-14","cited_by_count":25,"area":null},{"id":"2407.10194","title":"Curriculum Learning for Small Code Language Models","created":"2024-07-14","cited_by_count":22,"area":null},{"id":"2306.13421","title":"Retrieval-Pretrained Transformer: Long-range Language Modeling with Self-retrieval","created":"2023-06-23","cited_by_count":20,"area":"rag"},{"id":"2308.00189","title":"Generative Models as a Complex Systems Science: How can we make sense of large language model behavior?","created":"2023-07-31","cited_by_count":19,"area":null},{"id":"2204.10878","title":"ChapterBreak: A Challenge Dataset for Long-Range Language Models","created":"2022-04-22","cited_by_count":18,"area":null},{"id":"2210.12574","title":"The Curious Case of Absolute Position Embeddings","created":"2022-10-23","cited_by_count":17,"area":null},{"id":"2109.05941","title":"Efficient Contrastive Learning via Novel Data Augmentation and Curriculum Learning","created":"2021-09-10","cited_by_count":14,"area":null},{"id":"2212.03597","title":"DeepSpeed Data Efficiency: Improving Deep Learning Model Quality and Training Efficiency via Efficient Data Sampling and Routing","created":"2022-12-07","cited_by_count":13,"area":null},{"id":"2402.04286","title":"Progress and Opportunities of Foundation Models in Bioinformatics","created":"2024-02-06","cited_by_count":13,"area":null},{"id":"2411.04165","title":"Bio-xLSTM: Generative modeling, representation and in-context learning of biological and chemical sequences","created":"2024-11-06","cited_by_count":13,"area":null},{"id":"2311.04547","title":"Large GPT-like Models are Bad Babies: A Closer Look at the Relationship between Linguistic Competence and Psycholinguistic Measures","created":"2023-11-08","cited_by_count":11,"area":null},{"id":"2104.03474","title":"Revisiting Simple Neural Probabilistic Language Models","created":"2021-04-08","cited_by_count":10,"area":null},{"id":"2111.07819","title":"Testing the Generalization of Neural Language Models for COVID-19 Misinformation Detection","created":"2021-11-15","cited_by_count":10,"area":null},{"id":"2212.14815","title":"Black-box language model explanation by context length probing","created":"2022-12-30","cited_by_count":7,"area":null},{"id":"2410.00131","title":"Fisher Information-based Efficient Curriculum Federated Learning with Large Language Models","created":"2024-09-30","cited_by_count":6,"area":null},{"id":"2511.20102","title":"SSA: Sparse Sparse Attention by Aligning Full and Sparse Attention Outputs in Feature Space","created":"2025-11-25","cited_by_count":6,"area":null},{"id":"2203.10692","title":"Better Language Model with Hypernym Class Prediction","created":"2022-03-21","cited_by_count":5,"area":null},{"id":"2507.21449","title":"From Global to Local: A Scalable Benchmark for Local Posterior Sampling","created":"2025-07-29","cited_by_count":5,"area":null},{"id":"2203.12644","title":"Linearizing Transformer with Key-Value Memory","created":"2022-03-23","cited_by_count":4,"area":null},{"id":"2306.01705","title":"The Information Pathways Hypothesis: Transformers are Dynamic Self-Ensembles","created":"2023-06-02","cited_by_count":4,"area":null},{"id":"2411.03042","title":"Predictor-Corrector Enhanced Transformers with Exponential Moving Average Coefficient Learning","created":"2024-11-05","cited_by_count":4,"area":null},{"id":"2310.15672","title":"How Much Context Does My Attention-Based ASR System Need?","created":"2023-10-24","cited_by_count":3,"area":null},{"id":"2601.12499","title":"Failure Modes in Multi-Hop QA: The Weakest Link Effect and the Recognition Bottleneck","created":"2026-01-18","cited_by_count":3,"area":"lr"},{"id":"2205.04810","title":"The Importance of Context in Very Low Resource Language Modeling","created":"2022-05-10","cited_by_count":2,"area":null},{"id":"2312.10253","title":"Catwalk: A Unified Language Model Evaluation Framework for Many Datasets","created":"2023-12-15","cited_by_count":2,"area":null},{"id":"2204.08688","title":"DecBERT: Enhancing the Language Understanding of BERT with Causal Attention Masks","created":"2022-04-19","cited_by_count":1,"area":null},{"id":"2210.08458","title":"Learning Self-Regularized Adversarial Views for Self-Supervised Vision Transformers","created":"2022-10-16","cited_by_count":1,"area":null},{"id":"2306.16903","title":"Leveraging Cross-Utterance Context For ASR Decoding","created":"2023-06-29","cited_by_count":1,"area":null},{"id":"2310.12864","title":"The Locality and Symmetry of Positional Encodings","created":"2023-10-19","cited_by_count":1,"area":null},{"id":"2310.17591","title":"Lil-Bevo: Explorations of Strategies for Training Language Models in More Humanlike Ways","created":"2023-10-26","cited_by_count":1,"area":null},{"id":"2407.06533","title":"LETS-C: Leveraging Text Embedding for Time Series Classification","created":"2024-07-09","cited_by_count":1,"area":null},{"id":"2409.15697","title":"dnaGrinder: a lightweight and high-capacity genomic foundation model","created":"2024-09-24","cited_by_count":1,"area":null},{"id":"2502.20405","title":"Pause-Tuning for Long-Context Comprehension: A Lightweight Approach to LLM Attention Recalibration","created":"2025-02-01","cited_by_count":1,"area":null},{"id":"2506.14786","title":"PIPE: Physics-Informed Position Encoding for Alignment of Satellite Images and Time Series","created":"2025-05-27","cited_by_count":1,"area":null},{"id":"2510.23241","title":"Progressive Growing of Patch Size: Curriculum Learning for Accelerated and Improved Medical Image Segmentation","created":"2025-10-27","cited_by_count":1,"area":null},{"id":"2511.20849","title":"Length-MAX Tokenizer for Language Models","created":"2025-11-25","cited_by_count":1,"area":null},{"id":"2104.08742","title":"Go Forth and Prosper: Language Modeling with Ancient Textual History","created":"2021-04-18","cited_by_count":0,"area":null},{"id":"2111.02687","title":"CoreLM: Coreference-aware Language Model Fine-Tuning","created":"2021-11-04","cited_by_count":0,"area":null},{"id":"2310.14206","title":"Manifold-Preserving Transformers are Effective for Short-Long Range Encoding","created":"2023-10-22","cited_by_count":0,"area":null},{"id":"2312.03735","title":"Advancing State of the Art in Language Modeling","created":"2023-11-28","cited_by_count":0,"area":null},{"id":"2503.20103","title":"Bigger But Not Better: Small Neural Language Models Outperform Large Language Models in Detection of Thought Disorder","created":"2025-03-25","cited_by_count":0,"area":"lr"},{"id":"2602.14050","title":"Position Encoding with Random Float Sampling Enhances Length Generalization of Transformers","created":"2026-02-15","cited_by_count":0,"area":null},{"id":"2602.20296","title":"Learning to Solve Complex Problems via Dataset Decomposition","created":"2026-02-23","cited_by_count":0,"area":null},{"id":"2609.06218","title":"Branch-Centric Tokenization and Test-Time Augmentation for Skeleton Generation","created":"2026-09-05","cited_by_count":0,"area":null}],"next_cursor":null}