{"list":"other","total":63,"rows":[{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"2307.09288","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","created":"2023-07-18","cited_by_count":13028},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"1701.06538","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","created":"2017-01-23","cited_by_count":3509},{"id":"2101.03961","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","created":"2021-01-11","cited_by_count":3073},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"2203.05482","title":"Model soups: averaging weights of multiple fine-tuned models improves accuracy without increasing inference time","created":"2022-03-10","cited_by_count":1396},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"1906.02243","title":"Energy and Policy Considerations for Deep Learning in NLP","created":"2019-06-05","cited_by_count":1254},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"2302.05442","title":"Scaling Vision Transformers to 22 Billion Parameters","created":"2023-02-10","cited_by_count":694},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"1907.10597","title":"Green AI","created":"2019-07-22","cited_by_count":627},{"id":"1704.04760","title":"In-Datacenter Performance Analysis of a Tensor Processing Unit","created":"2017-04-16","cited_by_count":618},{"id":"2402.00838","title":"OLMo: Accelerating the Science of Language Models","created":"2024-02-01","cited_by_count":613},{"id":"1805.09767","title":"Local SGD Converges Fast and Communicates Little","created":"2018-05-24","cited_by_count":605},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245},{"id":"2111.00364","title":"Sustainable AI: Environmental Implications, Challenges and Opportunities","created":"2021-10-30","cited_by_count":242},{"id":"1911.02549","title":"MLPerf Inference Benchmark","created":"2019-11-06","cited_by_count":199},{"id":"2208.03306","title":"Branch-Train-Merge: Embarrassingly Parallel Training of Expert Language Models","created":"2022-08-05","cited_by_count":191},{"id":"2311.16863","title":"Power Hungry Processing: Watts Driving the Cost of AI Deployment?","created":"2023-11-28","cited_by_count":191},{"id":"2204.05149","title":"The Carbon Footprint of Machine Learning Training Will Plateau, Then Shrink","created":"2022-04-11","cited_by_count":184},{"id":"1807.05358","title":"Beyond Data and Model Parallelism for Deep Neural Networks","created":"2018-07-14","cited_by_count":179},{"id":"1811.02084","title":"Mesh-TensorFlow: Deep Learning for Supercomputers","created":"2018-11-05","cited_by_count":169},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126},{"id":"2311.08105","title":"DiLoCo: Distributed Low-Communication Training of Language Models","created":"2023-11-14","cited_by_count":121},{"id":"1806.03377","title":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","created":"2018-06-08","cited_by_count":117},{"id":"2207.10551","title":"Scaling Laws vs Model Architectures: How does Inductive Bias Influence Scaling?","created":"2022-07-21","cited_by_count":103},{"id":"2110.12894","title":"The Efficiency Misnomer","created":"2021-10-25","cited_by_count":98},{"id":"1910.02653","title":"Checkmate: Breaking the Memory Wall with Optimal Tensor Rematerialization","created":"2019-10-07","cited_by_count":91},{"id":"2006.15704","title":"PyTorch Distributed: Experiences on Accelerating Data Parallel Training","created":"2020-06-28","cited_by_count":90},{"id":"2406.19146","title":"Resolving Discrepancies in Compute-Optimal Scaling of Language Models","created":"2024-06-27","cited_by_count":88},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75},{"id":"2404.14619","title":"OpenELM: An Efficient Language Model Family with Open Training and Inference Framework","created":"2024-04-22","cited_by_count":75},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53},{"id":"2302.08476","title":"Counting Carbon: A Survey of Factors Influencing the Emissions of Machine Learning","created":"2023-02-16","cited_by_count":42},{"id":"1812.04056","title":"Accelerating Convolutional Neural Networks via Activation Map Compression","created":"2018-12-10","cited_by_count":34},{"id":"2206.11357","title":"GACT: Activation Compressed Training for Generic Network Architectures","created":"2022-06-22","cited_by_count":31},{"id":"2004.13336","title":"Automatic Cross-Replica Sharding of Weight Update in Data-Parallel Training","created":"2020-04-28","cited_by_count":30},{"id":"2411.01783","title":"Context Parallelism for Scalable Million-Token Inference","created":"2024-11-04","cited_by_count":26},{"id":"1712.04432","title":"Integrated Model, Batch and Domain Parallelism in Training Neural Networks","created":"2017-12-12","cited_by_count":16},{"id":"2310.02784","title":"MAD Max Beyond Single-Node: Enabling Large Machine Learning Model Acceleration on Distributed Systems","created":"2023-10-04","cited_by_count":14},{"id":"2302.06117","title":"The Framework Tax: Disparities Between Inference Efficiency in NLP Research and Deployment","created":"2023-02-13","cited_by_count":11},{"id":"2004.10856","title":"TensorOpt: Exploring the Tradeoffs in Distributed DNN Training with Auto-Parallelism","created":"2020-04-16","cited_by_count":10},{"id":"1907.13257","title":"Optimizing Multi-GPU Parallelization Strategies for Deep Learning Training","created":"2019-07-30","cited_by_count":9},{"id":"2311.05610","title":"Efficient Parallelization Layouts for Large-Scale Distributed Model Training","created":"2023-11-09","cited_by_count":8},{"id":"2406.05303","title":"Beyond Efficiency: Scaling AI Sustainably","created":"2024-06-08","cited_by_count":8},{"id":"2307.09701","title":"Efficiency Pentathlon: A Standardized Arena for Efficiency Evaluation","created":"2023-07-19","cited_by_count":7},{"id":"2211.03309","title":"DeepFlow: A Cross-Stack Pathfinding Framework for Distributed AI Systems","created":"2022-11-07","cited_by_count":6}],"next_cursor":null}