{"list":"other","total":69,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"2203.02155","title":"Training language models to follow instructions with human feedback","created":"2022-03-04","cited_by_count":15971},{"id":"1907.11692","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","created":"2019-07-26","cited_by_count":13372},{"id":"2103.14030","title":"Swin Transformer: Hierarchical Vision Transformer using Shifted Windows","created":"2021-03-25","cited_by_count":12139},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1910.13461","title":"BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension","created":"2019-10-29","cited_by_count":5893},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2210.11416","title":"Scaling Instruction-Finetuned Language Models","created":"2022-10-20","cited_by_count":3024},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"2407.10671","title":"Qwen2 Technical Report","created":"2024-07-15","cited_by_count":2120},{"id":"2303.03378","title":"PaLM-E: An Embodied Multimodal Language Model","created":"2023-03-06","cited_by_count":2071},{"id":"2211.05100","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","created":"2022-11-09","cited_by_count":1984},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2212.13138","title":"Large Language Models Encode Clinical Knowledge","created":"2022-12-26","cited_by_count":1830},{"id":"2009.14794","title":"Rethinking Attention with Performers","created":"2020-09-30","cited_by_count":1601},{"id":"2006.04768","title":"Linformer: Self-Attention with Linear Complexity","created":"2020-06-08","cited_by_count":1474},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"2111.09883","title":"Swin Transformer V2: Scaling Up Capacity and Resolution","created":"2021-11-18","cited_by_count":1228},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"1804.04235","title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost","created":"2018-04-11","cited_by_count":905},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"2202.03052","title":"OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework","created":"2022-02-07","cited_by_count":745},{"id":"2403.04652","title":"Yi: Open Foundation Models by 01.AI","created":"2024-03-07","cited_by_count":719},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"2006.10503","title":"SE(3)-Transformers: 3D Roto-Translation Equivariant Attention Networks","created":"2020-06-18","cited_by_count":607},{"id":"2210.10341","title":"BioGPT: Generative Pre-trained Transformer for Biomedical Text Generation and Mining","created":"2022-10-19","cited_by_count":538},{"id":"2407.08608","title":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","created":"2024-07-11","cited_by_count":519},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2302.06675","title":"Symbolic Discovery of Optimization Algorithms","created":"2023-02-13","cited_by_count":434},{"id":"2305.17333","title":"Fine-Tuning Language Models with Just Forward Passes","created":"2023-05-27","cited_by_count":345},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2301.10343","title":"ClimaX: A foundation model for weather and climate","created":"2023-01-24","cited_by_count":337},{"id":"2305.17888","title":"LLM-QAT: Data-Free Quantization Aware Training for Large Language Models","created":"2023-05-29","cited_by_count":324},{"id":"2402.17764","title":"The Era of 1-bit LLMs: All Large Language Models are in 1.58 Bits","created":"2024-02-27","cited_by_count":308},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302},{"id":"2307.14334","title":"Towards Generalist Biomedical AI","created":"2023-07-26","cited_by_count":289},{"id":"2311.16079","title":"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models","created":"2023-11-27","cited_by_count":274},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"2306.12873","title":"FuXi: A cascade machine learning forecasting system for 15-day global weather forecast","created":"2023-06-22","cited_by_count":241},{"id":"1905.12322","title":"A Study of BFLOAT16 for Deep Learning Training","created":"2019-05-29","cited_by_count":232},{"id":"2305.15075","title":"HuatuoGPT, towards Taming Language Model to Be a Doctor","created":"2023-05-24","cited_by_count":214},{"id":"2305.17100","title":"BiomedGPT: A Generalist Vision-Language Foundation Model for Diverse Biomedical Tasks","created":"2023-05-26","cited_by_count":201},{"id":"2310.11453","title":"BitNet: Scaling 1-bit Transformers for Large Language Models","created":"2023-10-17","cited_by_count":189},{"id":"2211.02556","title":"Pangu-Weather: A 3D High-Resolution Model for Fast and Accurate Global Weather Forecast","created":"2022-11-03","cited_by_count":166},{"id":"2112.05682","title":"Self-attention Does Not Need $O(n^2)$ Memory","created":"2021-12-10","cited_by_count":154},{"id":"2407.11062","title":"EfficientQAT: Efficient Quantization-Aware Training for Large Language Models","created":"2024-07-10","cited_by_count":127},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126},{"id":"2007.01045","title":"DAPPLE: A Pipelined Data Parallel Approach for Training Large Models","created":"2020-07-02","cited_by_count":101},{"id":"2110.14883","title":"Colossal-AI: A Unified Deep Learning System For Large-Scale Parallel Training","created":"2021-10-28","cited_by_count":96},{"id":"2403.14123","title":"AI and Memory Wall","created":"2024-03-21","cited_by_count":92},{"id":"2006.15704","title":"PyTorch Distributed: Experiences on Accelerating Data Parallel Training","created":"2020-06-28","cited_by_count":90},{"id":"2310.02025","title":"DeepZero: Scaling up Zeroth-Order Optimization for Deep Model Training","created":"2023-10-03","cited_by_count":76},{"id":"2306.05064","title":"K2: A Foundation Language Model for Geoscience Knowledge Understanding and Utilization","created":"2023-06-08","cited_by_count":69},{"id":"2401.06199","title":"xTrimoPGLM: Unified 100B-Scale Pre-trained Transformer for Deciphering the Language of Protein","created":"2024-01-11","cited_by_count":50},{"id":"2307.02047","title":"CAME: Confidence-guided Adaptive Memory Efficient Optimization","created":"2023-07-05","cited_by_count":36},{"id":"2312.15184","title":"ZO-AdaMU Optimizer: Adapting Perturbation by the Momentum and Uncertainty in Zeroth-order Optimization","created":"2023-12-23","cited_by_count":25},{"id":"2105.14450","title":"Maximizing Parallelism in Distributed Training for Huge Neural Networks","created":"2021-05-30","cited_by_count":23},{"id":"2410.07265","title":"A Survey: Collaborative Hardware and Software Design in the Era of Large Language Models","created":"2024-10-08","cited_by_count":23},{"id":"2301.11259","title":"Domain-Agnostic Molecular Generation with Chemical Feedback","created":"2023-01-26","cited_by_count":22},{"id":"2105.14500","title":"Tesseract: Parallelize the Tensor Parallelism Efficiently","created":"2021-05-30","cited_by_count":19},{"id":"2310.04610","title":"DeepSpeed4Science Initiative: Enabling Large-Scale Scientific Discovery through Sophisticated AI System Technologies","created":"2023-10-06","cited_by_count":15},{"id":"1807.02037","title":"TFLMS: Large Model Support in TensorFlow by Graph Rewriting","created":"2018-07-05","cited_by_count":12},{"id":"2405.15362","title":"Pipeline Parallelism with Controllable Memory","created":"2024-05-24","cited_by_count":11},{"id":"2404.11068","title":"ScaleFold: Reducing AlphaFold Initial Training Time to 10 Hours","created":"2024-04-17","cited_by_count":8}],"next_cursor":null}