{"total":354,"rows":[{"id":"2307.03172","title":"Lost in the Middle: How Language Models Use Long Contexts","created":"2023-07-06","cited_by_count":3455,"area":null},{"id":"2303.18223","title":"A Survey of Large Language Models","created":"2023-03-31","cited_by_count":2576,"area":null},{"id":"2103.03206","title":"Perceiver: General Perception with Iterative Attention","created":"2021-03-04","cited_by_count":1104,"area":null},{"id":"2412.06464","title":"Gated Delta Networks: Improving Mamba2 with Delta Rule","created":"2024-12-09","cited_by_count":532,"area":null},{"id":"2407.08608","title":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","created":"2024-07-11","cited_by_count":519,"area":null},{"id":"2312.06635","title":"Gated Linear Attention Transformers with Hardware-Efficient Training","created":"2023-12-11","cited_by_count":514,"area":"ssm"},{"id":"2106.04554","title":"A Survey of Transformers","created":"2021-06-08","cited_by_count":401,"area":null},{"id":"2406.06484","title":"Parallelizing Linear Transformers with the Delta Rule over Sequence Length","created":"2024-06-10","cited_by_count":396,"area":null},{"id":"2105.03824","title":"FNet: Mixing Tokens with Fourier Transforms","created":"2021-05-09","cited_by_count":347,"area":null},{"id":"2105.14995","title":"Choose a Transformer: Fourier or Galerkin","created":"2021-05-31","cited_by_count":343,"area":null},{"id":"2302.14376","title":"GNOT: A General Neural Operator Transformer for Operator Learning","created":"2023-02-28","cited_by_count":336,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2309.16039","title":"Effective Long-Context Scaling of Foundation Models","created":"2023-09-27","cited_by_count":303,"area":null},{"id":"2310.11454","title":"VeRA: Vector-based Random Matrix Adaptation","created":"2023-10-17","cited_by_count":300,"area":null},{"id":"2206.13947","title":"Long Range Language Modeling via Gated State Spaces","created":"2022-06-27","cited_by_count":265,"area":null},{"id":"2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","created":"2025-06-16","cited_by_count":263,"area":"lr"},{"id":"2111.13587","title":"Adaptive Fourier Neural Operators: Efficient Token Mixers for Transformers","created":"2021-11-24","cited_by_count":248,"area":null},{"id":"2103.06874","title":"CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation","created":"2021-03-11","cited_by_count":232,"area":null},{"id":"2112.07916","title":"LongT5: Efficient Text-To-Text Transformer for Long Sequences","created":"2021-12-15","cited_by_count":232,"area":null},{"id":"2202.08791","title":"cosFormer: Rethinking Softmax in Attention","created":"2022-02-17","cited_by_count":213,"area":"lr"},{"id":"2404.14294","title":"A Survey on Efficient Inference for Large Language Models","created":"2024-04-22","cited_by_count":205,"area":"inference"},{"id":"2202.10447","title":"Transformer Quality in Linear Time","created":"2022-02-21","cited_by_count":191,"area":null},{"id":"2207.02505","title":"Pure Transformers are Powerful Graph Learners","created":"2022-07-06","cited_by_count":188,"area":null},{"id":"2312.03863","title":"Efficient Large Language Models: A Survey","created":"2023-12-06","cited_by_count":186,"area":null},{"id":"2209.10655","title":"Mega: Moving Average Equipped Gated Attention","created":"2022-09-21","cited_by_count":179,"area":null},{"id":"2201.03533","title":"SCROLLS: Standardized CompaRison Over Long Language Sequences","created":"2022-01-10","cited_by_count":170,"area":null},{"id":"2307.01694","title":"Spike-driven Transformer","created":"2023-07-04","cited_by_count":146,"area":null},{"id":"2209.04881","title":"On The Computational Complexity of Self-Attention","created":"2022-09-11","cited_by_count":139,"area":null},{"id":"2110.11945","title":"SOFT: Softmax-free Transformer with Linear Complexity","created":"2021-10-22","cited_by_count":128,"area":null},{"id":"2404.07904","title":"HGRN2: Gated Linear RNNs with State Expansion","created":"2024-04-11","cited_by_count":127,"area":null},{"id":"2405.14366","title":"MiniCache: KV Cache Compression in Depth Dimension for Large Language Models","created":"2024-05-23","cited_by_count":125,"area":"inference"},{"id":"2401.06104","title":"Transformers are Multi-State RNNs","created":"2024-01-11","cited_by_count":120,"area":"inference"},{"id":"2402.04347","title":"The Hedgehog & the Porcupine: Expressive Linear Attentions with Softmax Mimicry","created":"2024-02-06","cited_by_count":120,"area":null},{"id":"2205.12674","title":"Training Language Models with Memory Augmentation","created":"2022-05-25","cited_by_count":113,"area":null},{"id":"2105.14103","title":"An Attention Free Transformer","created":"2021-05-28","cited_by_count":112,"area":null},{"id":"2203.07852","title":"Block-Recurrent Transformers","created":"2022-03-11","cited_by_count":110,"area":null},{"id":"2107.02192","title":"Long-Short Transformer: Efficient Transformers for Language and Vision","created":"2021-07-05","cited_by_count":102,"area":null},{"id":"2311.04823","title":"Hierarchically Gated Recurrent Neural Network for Sequence Modeling","created":"2023-11-08","cited_by_count":102,"area":null},{"id":"2210.10340","title":"The Devil in Linear Transformer","created":"2022-10-19","cited_by_count":98,"area":null},{"id":"2103.13076","title":"Finetuning Pretrained Transformers into RNNs","created":"2021-03-24","cited_by_count":96,"area":null},{"id":"2106.01540","title":"Luna: Linear Unified Nested Attention","created":"2021-06-03","cited_by_count":96,"area":null},{"id":"2106.06295","title":"Going Beyond Linear Transformers with Recurrent Fast Weight Programmers","created":"2021-06-11","cited_by_count":89,"area":null},{"id":"2210.09298","title":"What Makes Convolutional Models Great on Long Sequence Modeling?","created":"2022-10-17","cited_by_count":87,"area":null},{"id":"2205.09921","title":"KERPLE: Kernelized Relative Positional Embedding for Length Extrapolation","created":"2022-05-20","cited_by_count":86,"area":null},{"id":"2206.12126","title":"Temporal Attention Unit: Towards Efficient Spatiotemporal Predictive Learning","created":"2022-06-24","cited_by_count":86,"area":null},{"id":"2301.08721","title":"Batch Prompting: Efficient Inference with Large Language Model APIs","created":"2023-01-19","cited_by_count":85,"area":null},{"id":"2103.10619","title":"Scalable Vision Transformers with Hierarchical Pooling","created":"2021-03-19","cited_by_count":84,"area":null},{"id":"2205.13213","title":"Fast Vision Transformers with HiLo Attention","created":"2022-05-26","cited_by_count":78,"area":null},{"id":"2209.00099","title":"Efficient Methods for Natural Language Processing: A Survey","created":"2022-08-31","cited_by_count":78,"area":null},{"id":"2409.07146","title":"Gated Slot Attention for Efficient Linear-Time Sequence Modeling","created":"2024-09-11","cited_by_count":76,"area":null},{"id":"2402.09398","title":"Get More with LESS: Synthesizing Recurrence with KV Cache Compression for Efficient LLM Inference","created":"2024-02-14","cited_by_count":75,"area":"inference"},{"id":"2501.12352","title":"Test-time regression: a unifying framework for designing sequence models with associative memory","created":"2025-01-21","cited_by_count":74,"area":null},{"id":"2311.12351","title":"Advancing Transformer Architecture in Long-Context Large Language Models: A Comprehensive Survey","created":"2023-11-21","cited_by_count":72,"area":null},{"id":"2208.00748","title":"Efficient Long-Text Understanding with Short-Text Models","created":"2022-08-01","cited_by_count":71,"area":null},{"id":"2110.03860","title":"Token Pooling in Vision Transformers","created":"2021-10-08","cited_by_count":69,"area":null},{"id":"2202.06258","title":"Flowformer: Linearizing Transformers with Conservation Flows","created":"2022-02-13","cited_by_count":69,"area":null},{"id":"2107.05768","title":"Combiner: Full Attention Transformer with Sparse Computation Cost","created":"2021-07-12","cited_by_count":64,"area":null},{"id":"2106.06899","title":"Memory-efficient Transformers via Top-$k$ Attention","created":"2021-06-13","cited_by_count":60,"area":null},{"id":"2202.07765","title":"General-purpose, long-context autoregressive modeling with Perceiver AR","created":"2022-02-15","cited_by_count":60,"area":null},{"id":"2401.04658","title":"Lightning Attention-2: A Free Lunch for Handling Unlimited Sequence Lengths in Large Language Models","created":"2024-01-09","cited_by_count":57,"area":null},{"id":"2209.07484","title":"Hydra Attention: Efficient Attention with Many Heads","created":"2022-09-15","cited_by_count":55,"area":null},{"id":"2105.14217","title":"Less is More: Pay Less Attention in Vision Transformers","created":"2021-05-29","cited_by_count":53,"area":null},{"id":"2309.13345","title":"BAMBOO: A Comprehensive Benchmark for Evaluating Long Text Modeling Capacities of Large Language Models","created":"2023-09-23","cited_by_count":53,"area":null},{"id":"2503.02130","title":"Forgetting Transformer: Softmax Attention with a Forget Gate","created":"2025-03-03","cited_by_count":53,"area":null},{"id":"2406.16747","title":"Sparser is Faster and Less is More: Efficient Sparse Attention for Long-Range Transformers","created":"2024-06-24","cited_by_count":52,"area":null},{"id":"2106.12566","title":"Stable, Fast and Accurate: Kernelized Attention with Relative Positional Encoding","created":"2021-06-23","cited_by_count":51,"area":null},{"id":"2405.06640","title":"Linearizing Large Language Models","created":"2024-05-10","cited_by_count":51,"area":null},{"id":"2210.10349","title":"Museformer: Transformer with Fine- and Coarse-Grained Attention for Music Generation","created":"2022-10-19","cited_by_count":49,"area":null},{"id":"2406.03372","title":"Training of Physical Neural Networks","created":"2024-06-05","cited_by_count":49,"area":null},{"id":"2406.04207","title":"CDMamba: Incorporating Local Clues into Mamba for Remote Sensing Image Binary Change Detection","created":"2024-06-06","cited_by_count":48,"area":"ssm"},{"id":"2302.14502","title":"A Survey on Long Text Modeling with Transformers","created":"2023-02-28","cited_by_count":47,"area":null},{"id":"2506.05233","title":"MesaNet: Sequence Modeling by Locally Optimal Test-Time Training","created":"2025-06-05","cited_by_count":47,"area":null},{"id":"2202.05798","title":"The Dual Form of Neural Networks Revisited: Connecting Test Time Predictions to Training Patterns via Spotlights of Attention","created":"2022-02-11","cited_by_count":46,"area":null},{"id":"2402.05964","title":"A Survey on Transformer Compression","created":"2024-02-05","cited_by_count":46,"area":null},{"id":"2402.03170","title":"Is Mamba Capable of In-Context Learning?","created":"2024-02-05","cited_by_count":45,"area":"ssm"},{"id":"2210.10890","title":"Mitigating spectral bias for the multiscale operator learning","created":"2022-10-19","cited_by_count":44,"area":null},{"id":"2409.02097","title":"LinFusion: 1 GPU, 1 Minute, 16K Image","created":"2024-09-03","cited_by_count":43,"area":"diffusion"},{"id":"2202.05780","title":"A Modern Self-Referential Weight Matrix That Learns to Modify Itself","created":"2022-02-11","cited_by_count":41,"area":null},{"id":"2302.04931","title":"In-Context Learning with Many Demonstration Examples","created":"2023-02-09","cited_by_count":39,"area":null},{"id":"2107.07999","title":"From block-Toeplitz matrices to differential equations on graphs: towards a general theory for scalable masked Transformers","created":"2021-07-16","cited_by_count":38,"area":null},{"id":"2105.14528","title":"Fast Nearest Neighbor Machine Translation","created":"2021-05-30","cited_by_count":37,"area":null},{"id":"2204.07756","title":"Visual Attention Methods in Deep Learning: An In-Depth Survey","created":"2022-04-16","cited_by_count":37,"area":null},{"id":"2212.08136","title":"Efficient Long Sequence Modeling via State Space Augmented Transformer","created":"2022-12-15","cited_by_count":37,"area":null},{"id":"2110.02488","title":"ABC: Attention with Bounded-memory Control","created":"2021-10-06","cited_by_count":36,"area":null},{"id":"2310.01655","title":"PolySketchFormer: Fast Transformers via Sketching Polynomial Kernels","created":"2023-10-02","cited_by_count":36,"area":null},{"id":"2312.07987","title":"SwitchHead: Accelerating Transformers with Mixture-of-Experts Attention","created":"2023-12-13","cited_by_count":34,"area":"moe"},{"id":"2110.06821","title":"Leveraging redundancy in attention with Reuse Transformers","created":"2021-10-13","cited_by_count":33,"area":null},{"id":"2105.06548","title":"Not All Memories are Created Equal: Learning to Forget by Expiring","created":"2021-05-13","cited_by_count":32,"area":null},{"id":"2112.04139","title":"Bidimensional Leaderboards: Generate and Evaluate Language Hand in Hand","created":"2021-12-08","cited_by_count":32,"area":null},{"id":"2206.08898","title":"SimA: Simple Softmax-free Attention for Vision Transformers","created":"2022-06-17","cited_by_count":32,"area":null},{"id":"2109.15142","title":"Redesigning the Transformer Architecture with Insights from Multi-particle Dynamical Systems","created":"2021-09-30","cited_by_count":31,"area":null},{"id":"2204.04667","title":"Linear Complexity Randomized Self-attention Mechanism","created":"2022-04-10","cited_by_count":31,"area":null},{"id":"2109.05019","title":"Spike2Vec: An Efficient and Scalable Embedding Approach for COVID-19 Spike Sequences","created":"2021-09-12","cited_by_count":30,"area":null},{"id":"2204.13154","title":"Attention Mechanism in Neural Networks: Where it Comes and Where it Goes","created":"2022-04-27","cited_by_count":29,"area":null},{"id":"2305.19798","title":"Primal-Attention: Self-attention through Asymmetric Kernel SVD in Primal Representation","created":"2023-05-31","cited_by_count":29,"area":null},{"id":"2110.08678","title":"Improving Transformers with Probabilistic Attention Keys","created":"2021-10-16","cited_by_count":28,"area":null},{"id":"2108.02347","title":"FMMformer: Efficient and Flexible Transformer via Decomposed Near-field and Far-field Attention","created":"2021-08-05","cited_by_count":26,"area":null},{"id":"2210.04243","title":"Fine-Tuning Pre-trained Transformers into Decaying Fast Weights","created":"2022-10-09","cited_by_count":26,"area":null},{"id":"2304.03589","title":"On Efficient Training of Large-Scale Deep Learning Models: A Literature Review","created":"2023-04-07","cited_by_count":26,"area":null},{"id":"2309.16354","title":"Transformer-VQ: Linear-Time Transformers via Vector Quantization","created":"2023-09-28","cited_by_count":26,"area":null}],"next_cursor":"100"}