{"list":"other","total":64,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"2110.14168","title":"Training Verifiers to Solve Math Word Problems","created":"2021-10-27","cited_by_count":10109},{"id":"2107.03374","title":"Evaluating Large Language Models Trained on Code","created":"2021-07-07","cited_by_count":8320},{"id":"2009.03300","title":"Measuring Massive Multitask Language Understanding","created":"2020-09-07","cited_by_count":7963},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2312.00752","title":"Mamba: Linear-Time Sequence Modeling with Selective State Spaces","created":"2023-12-01","cited_by_count":4718},{"id":"2104.09864","title":"RoFormer: Enhanced Transformer with Rotary Position Embedding","created":"2021-04-20","cited_by_count":4703},{"id":"1905.07830","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","created":"2019-05-19","cited_by_count":4100},{"id":"2309.17453","title":"Efficient Streaming Language Models with Attention Sinks","created":"2023-09-29","cited_by_count":2186},{"id":"2002.05202","title":"GLU Variants Improve Transformer","created":"2020-02-12","cited_by_count":1804},{"id":"1910.07467","title":"Root Mean Square Layer Normalization","created":"2019-10-16","cited_by_count":1480},{"id":"2305.13245","title":"GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints","created":"2023-05-22","cited_by_count":1346},{"id":"2404.06654","title":"RULER: What's the Real Context Size of Your Long-Context Language Models?","created":"2024-04-09","cited_by_count":1346},{"id":"2405.21060","title":"Transformers are SSMs: Generalized Models and Efficient Algorithms Through Structured State Space Duality","created":"2024-05-31","cited_by_count":1336},{"id":"2209.11895","title":"In-context Learning and Induction Heads","created":"2022-09-24","cited_by_count":964},{"id":"2309.16588","title":"Vision Transformers Need Registers","created":"2023-09-28","cited_by_count":871},{"id":"1905.09418","title":"Analyzing Multi-Head Self-Attention: Specialized Heads Do the Heavy Lifting, the Rest Can Be Pruned","created":"2019-05-23","cited_by_count":856},{"id":"2210.02414","title":"GLM-130B: An Open Bilingual Pre-trained Model","created":"2022-10-05","cited_by_count":843},{"id":"2306.15595","title":"Extending Context Window of Large Language Models via Positional Interpolation","created":"2023-06-27","cited_by_count":740},{"id":"2305.08322","title":"C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models","created":"2023-05-15","cited_by_count":708},{"id":"2105.13290","title":"CogView: Mastering Text-to-Image Generation via Transformers","created":"2021-05-26","cited_by_count":694},{"id":"2307.08621","title":"Retentive Network: A Successor to Transformer for Large Language Models","created":"2023-07-17","cited_by_count":613},{"id":"2309.00071","title":"YaRN: Efficient Context Window Extension of Large Language Models","created":"2023-08-31","cited_by_count":601},{"id":"2412.06464","title":"Gated Delta Networks: Improving Mamba2 with Delta Rule","created":"2024-12-09","cited_by_count":532},{"id":"2502.11089","title":"Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention","created":"2025-02-16","cited_by_count":407},{"id":"1402.1869","title":"On the Number of Linear Regions of Deep Neural Networks","created":"2014-02-08","cited_by_count":381},{"id":"2402.17762","title":"Massive Activations in Large Language Models","created":"2024-02-27","cited_by_count":294},{"id":"1908.06954","title":"Attention on Attention for Image Captioning","created":"2019-08-19","cited_by_count":289},{"id":"1812.06162","title":"An Empirical Model of Large-Batch Training","created":"2018-12-14","cited_by_count":281},{"id":"1505.00387","title":"Highway Networks","created":"2015-05-03","cited_by_count":260},{"id":"1701.05923","title":"Gate-Variants of Gated Recurrent Unit (GRU) Neural Networks","created":"2017-01-20","cited_by_count":242},{"id":"2012.09852","title":"SpAtten: Efficient Sparse Attention Architecture with Cascade Token and Head Pruning","created":"2020-12-17","cited_by_count":234},{"id":"2410.10781","title":"When Attention Sink Emerges in Language Models: An Empirical View","created":"2024-10-14","cited_by_count":208},{"id":"2202.10447","title":"Transformer Quality in Linear Time","created":"2022-02-21","cited_by_count":191},{"id":"2203.00555","title":"DeepNet: Scaling Transformers to 1,000 Layers","created":"2022-03-01","cited_by_count":178},{"id":"1803.07294","title":"GaAN: Gated Attention Networks for Learning on Large and Spatiotemporal Graphs","created":"2018-03-20","cited_by_count":165},{"id":"2306.12929","title":"Quantizable Transformers: Removing Outliers by Helping Attention Heads Do Nothing","created":"2023-06-22","cited_by_count":151},{"id":"2411.13676","title":"Hymba: A Hybrid-head Architecture for Small Language Models","created":"2024-11-20","cited_by_count":99},{"id":"2406.15765","title":"Unveiling and Harnessing Hidden Attention Sinks: Enhancing Large Language Models without Training through Attention Calibration","created":"2024-06-22","cited_by_count":74},{"id":"2310.04415","title":"Why Do We Need Weight Decay in Modern Deep Learning?","created":"2023-10-06","cited_by_count":72},{"id":"2409.04431","title":"Theory, Analysis, and Best Practices for Sigmoid Self-Attention","created":"2024-09-06","cited_by_count":64},{"id":"1912.11637","title":"Explicit Sparse Transformer: Concentrated Attention Through Explicit Selection","created":"2019-12-25","cited_by_count":62},{"id":"2003.02436","title":"Talking-Heads Attention","created":"2020-03-05","cited_by_count":62},{"id":"2405.16039","title":"MoEUT: Mixture-of-Experts Universal Transformers","created":"2024-05-25","cited_by_count":59},{"id":"2401.04658","title":"Lightning Attention-2: A Free Lunch for Handling Unlimited Sequence Lengths in Large Language Models","created":"2024-01-09","cited_by_count":57},{"id":"2501.11873","title":"Demons in the Detail: On Implementing Load Balancing Loss for Training Specialized Mixture-of-Expert Models","created":"2025-01-21","cited_by_count":54},{"id":"2503.02130","title":"Forgetting Transformer: Softmax Attention with a Forget Gate","created":"2025-03-03","cited_by_count":53},{"id":"2312.16903","title":"Spike No More: Stabilizing the Pre-training of Large Language Models","created":"2023-12-28","cited_by_count":52},{"id":"2312.07987","title":"SwitchHead: Accelerating Transformers with Mixture-of-Experts Attention","created":"2023-12-13","cited_by_count":34},{"id":"2405.17381","title":"Various Lengths, Constant Speed: Efficient Language Modeling with Lightning Attention","created":"2024-05-27","cited_by_count":31},{"id":"2310.10837","title":"Approximating Two-Layer Feedforward Networks for Efficient Transformers","created":"2023-10-16","cited_by_count":27},{"id":"2504.20966","title":"Softpick: No Attention Sink, No Massive Activations with Rectified Softmax","created":"2025-04-29","cited_by_count":22},{"id":"2411.12892","title":"Selective Attention: Enhancing Transformer through Principled Context Control","created":"2024-11-19","cited_by_count":16},{"id":"2503.08908","title":"Interpreting the Repeated Token Phenomenon in Large Language Models","created":"2025-03-11","cited_by_count":16},{"id":"2207.14000","title":"Multi-Step Deductive Reasoning Over Natural Language: An Empirical Study on Out-of-Distribution Generalisation","created":"2022-07-28","cited_by_count":15},{"id":"2502.07365","title":"LongReD: Mitigating Short-Text Degradation of Long-Context Large Language Models via Restoration Distillation","created":"2025-02-11","cited_by_count":14},{"id":"2505.00315","title":"Mixture of Sparse Attention: Content-Based Learnable Sparse Attention via Expert-Choice Routing","created":"2025-05-01","cited_by_count":14},{"id":"1912.00349","title":"Not All Attention Is Needed: Gated Attention Network for Sequence Data","created":"2019-12-01","cited_by_count":13},{"id":"2405.08553","title":"Improving Transformers with Dynamically Composable Multi-Head Attention","created":"2024-05-14","cited_by_count":10},{"id":"2504.00927","title":"Multi-Token Attention","created":"2025-04-01","cited_by_count":10},{"id":"2305.15033","title":"SmartTrim: Adaptive Tokens and Attention Pruning for Efficient Vision-Language Models","created":"2023-05-24","cited_by_count":7},{"id":"2410.07145","title":"Stuffed Mamba: Oversized States Lead to the Inability to Forget","created":"2024-10-09","cited_by_count":7},{"id":"2407.04400","title":"Hard-Attention Gates with Gradient Routing for Endoscopic Image Computing","created":"2024-07-05","cited_by_count":3}],"next_cursor":null}