{"list":"other","total":58,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1803.05457","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","created":"2018-03-14","cited_by_count":4886},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"1905.07830","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","created":"2019-05-19","cited_by_count":4100},{"id":"2004.05150","title":"Longformer: The Long-Document Transformer","created":"2020-04-10","cited_by_count":3413},{"id":"1911.11641","title":"PIQA: Reasoning about Physical Commonsense in Natural Language","created":"2019-11-26","cited_by_count":2883},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"2309.17453","title":"Efficient Streaming Language Models with Attention Sinks","created":"2023-09-29","cited_by_count":2186},{"id":"1809.02789","title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","created":"2018-09-08","cited_by_count":2164},{"id":"2007.14062","title":"Big Bird: Transformers for Longer Sequences","created":"2020-07-28","cited_by_count":1684},{"id":"2404.06654","title":"RULER: What's the Real Context Size of Your Long-Context Language Models?","created":"2024-04-09","cited_by_count":1346},{"id":"2406.17557","title":"The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale","created":"2024-06-25","cited_by_count":1160},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103},{"id":"1606.06031","title":"The LAMBADA dataset: Word prediction requiring a broad discourse context","created":"2016-06-20","cited_by_count":992},{"id":"2002.04745","title":"On Layer Normalization in the Transformer Architecture","created":"2020-02-12","cited_by_count":922},{"id":"2006.05205","title":"On the Bottleneck of Graph Neural Networks and its Practical Implications","created":"2020-06-09","cited_by_count":631},{"id":"2309.00071","title":"YaRN: Efficient Context Window Extension of Large Language Models","created":"2023-08-31","cited_by_count":601},{"id":"2305.18654","title":"Faith and Fate: Limits of Transformers on Compositionality","created":"2023-05-29","cited_by_count":503},{"id":"2103.03404","title":"Attention is Not All You Need: Pure Attention Loses Rank Doubly Exponentially with Depth","created":"2021-03-05","cited_by_count":419},{"id":"2407.02490","title":"MInference 1.0: Accelerating Pre-filling for Long-Context LLMs via Dynamic Sparse Attention","created":"2024-07-02","cited_by_count":383},{"id":"1602.02068","title":"From Softmax to Sparsemax: A Sparse Model of Attention and Multi-Label Classification","created":"2016-02-05","cited_by_count":341},{"id":"2305.19466","title":"The Impact of Positional Encoding on Length Generalization in Transformers","created":"2023-05-31","cited_by_count":327},{"id":"2309.16039","title":"Effective Long-Context Scaling of Foundation Models","created":"2023-09-27","cited_by_count":303},{"id":"2402.01032","title":"Repeat After Me: Transformers are Better than State Space Models at Copying","created":"2024-02-01","cited_by_count":216},{"id":"2305.16300","title":"Landmark Attention: Random-Access Infinite Context Length for Transformers","created":"2023-05-25","cited_by_count":213},{"id":"2207.04901","title":"Exploring Length Generalization in Large Language Models","created":"2022-07-11","cited_by_count":206},{"id":"2312.04927","title":"Zoology: Measuring and Improving Recall in Efficient Language Models","created":"2023-12-08","cited_by_count":204},{"id":"2310.16028","title":"What Algorithms can Transformers Learn? A Study in Length Generalization","created":"2023-10-24","cited_by_count":199},{"id":"1905.05702","title":"Sparse Sequence-to-Sequence Models","created":"2019-05-14","cited_by_count":170},{"id":"2303.06296","title":"Stabilizing Transformer Training by Preventing Attention Entropy Collapse","created":"2023-03-11","cited_by_count":161},{"id":"1909.00015","title":"Adaptively Sparse Transformers","created":"2019-08-30","cited_by_count":155},{"id":"2206.03126","title":"Signal Propagation in Transformers: Theoretical Perspectives and the Role of Rank Collapse","created":"2022-06-07","cited_by_count":141},{"id":"2503.16428","title":"XAttention: Block Sparse Attention with Antidiagonal Scoring","created":"2025-03-20","cited_by_count":137},{"id":"2502.20766","title":"FlexPrefill: A Context-Aware Sparse Attention Mechanism for Efficient Long-Sequence Inference","created":"2025-02-28","cited_by_count":123},{"id":"1901.02324","title":"Learning with Fenchel-Young Losses","created":"2019-01-08","cited_by_count":119},{"id":"2410.06205","title":"Round and Round We Go! What makes Rotary Positional Encodings useful?","created":"2024-10-08","cited_by_count":115},{"id":"2406.04267","title":"Transformers need glasses! Information over-squashing in language tasks","created":"2024-06-06","cited_by_count":91},{"id":"2205.09921","title":"KERPLE: Kernelized Relative Positional Embedding for Length Extrapolation","created":"2022-05-20","cited_by_count":86},{"id":"2410.06511","title":"TorchTitan: One-stop PyTorch native solution for production ready LLM pre-training","created":"2024-10-09","cited_by_count":85},{"id":"2306.00946","title":"Exposing Attention Glitches with Flip-Flop Language Modeling","created":"2023-06-01","cited_by_count":75},{"id":"2306.15400","title":"Length Generalization in Arithmetic Transformers","created":"2023-06-27","cited_by_count":66},{"id":"2106.06899","title":"Memory-efficient Transformers via Top-$k$ Attention","created":"2021-06-13","cited_by_count":60},{"id":"2502.01951","title":"On the Emergence of Position Bias in Transformers","created":"2025-02-04","cited_by_count":59},{"id":"2410.01104","title":"Softmax is not Enough (for Sharp Size Generalisation)","created":"2024-10-01","cited_by_count":46},{"id":"2501.19399","title":"Scalable-Softmax Is Superior for Attention","created":"2025-01-31","cited_by_count":45},{"id":"2502.10818","title":"On Vanishing Gradients, Over-Smoothing, and Over-Squashing in GNNs: Bridging Recurrent and Graph Learning","created":"2025-02-15","cited_by_count":43},{"id":"2401.07004","title":"Extending LLMs' Context Window with 100 Samples","created":"2024-01-13","cited_by_count":21},{"id":"2108.01988","title":"Sparse Continuous Distributions and Fenchel-Young Losses","created":"2021-08-04","cited_by_count":14},{"id":"2109.12188","title":"Predicting Attention Sparsity in Transformers","created":"2021-09-24","cited_by_count":13},{"id":"2410.17980","title":"Scaling Stick-Breaking Attention: An Efficient Implementation and In-depth Study","created":"2024-10-23","cited_by_count":13},{"id":"2502.12082","title":"AdaSplash: Adaptive Sparse Flash Attention","created":"2025-02-17","cited_by_count":12},{"id":"2412.12465","title":"Core Context Aware Transformers for Long Context Language Modeling","created":"2024-12-17","cited_by_count":11},{"id":"2410.01651","title":"Efficient Length-Generalizable Attention via Causal Retrieval for Long-Context Language Modeling","created":"2024-10-02","cited_by_count":9},{"id":"2501.14577","title":"ZETA: Leveraging Z-order Curves for Efficient Top-k Attention","created":"2025-01-24","cited_by_count":9},{"id":"2505.17083","title":"Scale-invariant Attention","created":"2025-05-20","cited_by_count":5},{"id":"2505.22107","title":"Curse of High Dimensionality Issue in Transformer for Long-context Modeling","created":"2025-05-28","cited_by_count":5},{"id":"2501.08570","title":"Information Entropy Invariance: Enhancing Length Extrapolation in Attention Mechanisms","created":"2025-01-15","cited_by_count":4}],"next_cursor":null}