{"list":"other","total":72,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2505.09388","title":"Qwen3 Technical Report","created":"2025-05-14","cited_by_count":8820},{"id":"1607.06450","title":"Layer Normalization","created":"2016-07-21","cited_by_count":5669},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"1502.01852","title":"Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification","created":"2015-02-06","cited_by_count":3611},{"id":"2004.05150","title":"Longformer: The Long-Document Transformer","created":"2020-04-10","cited_by_count":3413},{"id":"2310.06825","title":"Mistral 7B","created":"2023-10-10","cited_by_count":3164},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"2309.17453","title":"Efficient Streaming Language Models with Attention Sinks","created":"2023-09-29","cited_by_count":2186},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"1904.10509","title":"Generating Long Sequences with Sparse Transformers","created":"2019-04-23","cited_by_count":1546},{"id":"1910.07467","title":"Root Mean Square Layer Normalization","created":"2019-10-16","cited_by_count":1480},{"id":"2404.06654","title":"RULER: What's the Real Context Size of Your Long-Context Language Models?","created":"2024-04-09","cited_by_count":1346},{"id":"2405.21060","title":"Transformers are SSMs: Generalized Models and Efficient Algorithms Through Structured State Space Duality","created":"2024-05-31","cited_by_count":1336},{"id":"1803.05407","title":"Averaging Weights Leads to Wider Optima and Better Generalization","created":"2018-03-14","cited_by_count":1209},{"id":"1803.02155","title":"Self-Attention with Relative Position Representations","created":"2018-03-06","cited_by_count":1120},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103},{"id":"2306.15595","title":"Extending Context Window of Large Language Models via Positional Interpolation","created":"2023-06-27","cited_by_count":740},{"id":"2309.00071","title":"YaRN: Efficient Context Window Extension of Large Language Models","created":"2023-08-31","cited_by_count":601},{"id":"1706.02515","title":"Self-Normalizing Neural Networks","created":"2017-06-08","cited_by_count":598},{"id":"1705.03122","title":"Convolutional Sequence to Sequence Learning","created":"2017-05-08","cited_by_count":552},{"id":"2305.19466","title":"The Impact of Positional Encoding on Length Generalization in Transformers","created":"2023-05-31","cited_by_count":327},{"id":"2402.13753","title":"LongRoPE: Extending LLM Context Window Beyond 2 Million Tokens","created":"2024-02-21","cited_by_count":313},{"id":"2309.16039","title":"Effective Long-Context Scaling of Foundation Models","created":"2023-09-27","cited_by_count":303},{"id":"2402.01032","title":"Repeat After Me: Transformers are Better than State Space Models at Copying","created":"2024-02-01","cited_by_count":216},{"id":"2207.04901","title":"Exploring Length Generalization in Large Language Models","created":"2022-07-11","cited_by_count":206},{"id":"2312.04927","title":"Zoology: Measuring and Improving Recall in Efficient Language Models","created":"2023-12-08","cited_by_count":204},{"id":"2402.10171","title":"Data Engineering for Scaling Language Models to 128K Context","created":"2024-02-15","cited_by_count":186},{"id":"2406.07887","title":"An Empirical Study of Mamba-based Language Models","created":"2024-06-12","cited_by_count":186},{"id":"2410.02660","title":"How to Train Long-Context Language Models (Effectively)","created":"2024-10-03","cited_by_count":146},{"id":"2401.01325","title":"LLM Maybe LongLM: Self-Extend LLM Context Window Without Tuning","created":"2024-01-02","cited_by_count":136},{"id":"2212.10554","title":"A Length-Extrapolatable Transformer","created":"2022-12-20","cited_by_count":127},{"id":"2310.05209","title":"Scaling Laws of RoPE-based Extrapolation","created":"2023-10-08","cited_by_count":125},{"id":"2308.16137","title":"LM-Infinite: Zero-Shot Extreme Length Generalization for Large Language Models","created":"2023-08-30","cited_by_count":124},{"id":"2305.16843","title":"Randomized Positional Encodings Boost Length Generalization of Transformers","created":"2023-05-26","cited_by_count":117},{"id":"2202.12172","title":"Overcoming a Theoretical Limitation of Self-Attention","created":"2022-02-24","cited_by_count":113},{"id":"2309.10400","title":"PoSE: Efficient Context Window Extension of LLMs via Positional Skip-wise Training","created":"2023-09-19","cited_by_count":111},{"id":"1307.1493","title":"Dropout Training as Adaptive Regularization","created":"2013-07-04","cited_by_count":93},{"id":"2406.04267","title":"Transformers need glasses! Information over-squashing in language tasks","created":"2024-06-06","cited_by_count":91},{"id":"2205.09921","title":"KERPLE: Kernelized Relative Positional Embedding for Length Extrapolation","created":"2022-05-20","cited_by_count":86},{"id":"2402.09371","title":"Transformers Can Achieve Length Generalization But Not Robustly","created":"2024-02-14","cited_by_count":86},{"id":"2410.06511","title":"TorchTitan: One-stop PyTorch native solution for production ready LLM pre-training","created":"2024-10-09","cited_by_count":85},{"id":"2411.18704","title":"Exponential Moving Average of Weights in Deep Learning: Dynamics and Benefits","created":"2024-11-27","cited_by_count":71},{"id":"2410.18745","title":"Why Does the Effective Context Length of LLMs Fall Short?","created":"2024-10-24","cited_by_count":63},{"id":"2402.17463","title":"Training-Free Long-Context Scaling of Large Language Models","created":"2024-02-27","cited_by_count":62},{"id":"2405.18719","title":"Contextual Position Encoding: Learning to Count What's Important","created":"2024-05-29","cited_by_count":54},{"id":"2405.14591","title":"Base of RoPE Bounds Context Length","created":"2024-05-23","cited_by_count":52},{"id":"2406.14528","title":"DeciMamba: Exploring the Length Extrapolation Potential of Mamba","created":"2024-06-20","cited_by_count":49},{"id":"2410.01104","title":"Softmax is not Enough (for Sharp Size Generalisation)","created":"2024-10-01","cited_by_count":46},{"id":"2501.19399","title":"Scalable-Softmax Is Superior for Attention","created":"2025-01-31","cited_by_count":45},{"id":"2310.16450","title":"CLEX: Continuous Length Extrapolation for Large Language Models","created":"2023-10-25","cited_by_count":44},{"id":"2212.10356","title":"Dissecting Transformer Length Extrapolation via the Lens of Receptive Field Analysis","created":"2022-12-20","cited_by_count":42},{"id":"2504.16053","title":"LongMamba: Enhancing Mamba's Long Context Capabilities via Training-Free Receptive Field Enlargement","created":"2025-04-22","cited_by_count":42},{"id":"2412.17739","title":"Fourier Position Embedding: Enhancing Attention's Periodic Extension for Length Generalization","created":"2024-12-23","cited_by_count":39},{"id":"1801.05134","title":"Understanding the Disharmony between Dropout and Batch Normalization by Variance Shift","created":"2018-01-16","cited_by_count":37},{"id":"2109.07276","title":"Sequence Length is a Domain: Length-based Overfitting in Transformer Models","created":"2021-09-15","cited_by_count":35},{"id":"2505.16381","title":"PaTH Attention: Position Encoding via Accumulating Householder Transformations","created":"2025-05-22","cited_by_count":34},{"id":"2405.14722","title":"DAPE: Data-Adaptive Positional Encoding for Length Extrapolation","created":"2024-05-23","cited_by_count":31},{"id":"2506.05209","title":"The Common Pile v0.1: An 8TB Dataset of Public Domain and Openly Licensed Text","created":"2025-06-05","cited_by_count":31},{"id":"2403.00071","title":"Resonance RoPE: Improving Context Length Generalization of Large Language Models","created":"2024-02-29","cited_by_count":27},{"id":"2401.16421","title":"Two Stones Hit One Bird: Bilevel Positional Encoding for Better Length Extrapolation","created":"2024-01-29","cited_by_count":24},{"id":"2507.02782","title":"Understanding and Improving Length Generalization in Recurrent Models","created":"2025-07-03","cited_by_count":23},{"id":"2506.16640","title":"Long-Context Generalization with Sparse Attention","created":"2025-06-19","cited_by_count":21},{"id":"2509.19633","title":"Mamba Modulation: On the Length Generalization of Mamba","created":"2025-09-23","cited_by_count":16},{"id":"2104.04946","title":"UniDrop: A Simple yet Effective Technique to Improve Transformer without Extra Cost","created":"2021-04-11","cited_by_count":11},{"id":"2407.05489","title":"How Effective are State Space Models for Machine Translation?","created":"2024-07-07","cited_by_count":6},{"id":"2505.07793","title":"Overflow Prevention Enhances Long-Context Recurrent LLMs","created":"2025-05-12","cited_by_count":6},{"id":"2506.11769","title":"Long-Short Alignment for Effective Long-Context Modeling in LLMs","created":"2025-06-13","cited_by_count":4},{"id":"2510.14826","title":"To Infinity and Beyond: Tool-Use Unlocks Length Generalization in State Space Models","created":"2025-10-16","cited_by_count":4},{"id":"2410.15859","title":"Mesa-Extrapolation: A Weave Position Encoding Method for Enhanced Extrapolation in LLMs","created":"2024-10-21","cited_by_count":2}],"next_cursor":null}