{"total":183,"rows":[{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326,"area":null},{"id":"2112.04482","title":"FLAVA: A Foundational Language And Vision Alignment Model","created":"2021-12-08","cited_by_count":649,"area":null},{"id":"2111.09296","title":"XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale","created":"2021-11-17","cited_by_count":601,"area":null},{"id":"2303.06865","title":"FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU","created":"2023-03-13","cited_by_count":575,"area":null},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559,"area":null},{"id":"2305.13516","title":"Scaling Speech Technology to 1,000+ Languages","created":"2023-05-22","cited_by_count":411,"area":null},{"id":"2207.00032","title":"DeepSpeed Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale","created":"2022-06-30","cited_by_count":407,"area":null},{"id":"2110.02861","title":"8-bit Optimizers via Block-wise Quantization","created":"2021-10-06","cited_by_count":403,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2502.05564","title":"TabICL: A Tabular Foundation Model for In-Context Learning on Large Data","created":"2025-02-08","cited_by_count":289,"area":null},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245,"area":null},{"id":"2111.00364","title":"Sustainable AI: Environmental Implications, Challenges and Opportunities","created":"2021-10-30","cited_by_count":242,"area":null},{"id":"2308.00708","title":"VeriGen: A Large Language Model for Verilog Code Generation","created":"2023-07-28","cited_by_count":200,"area":null},{"id":"2312.03863","title":"Efficient Large Language Models: A Survey","created":"2023-12-06","cited_by_count":186,"area":null},{"id":"2310.05694","title":"A Survey of Large Language Models for Healthcare: from Data, Technology, and Applications to Accountability and Ethics","created":"2023-10-09","cited_by_count":163,"area":null},{"id":"2406.16793","title":"Adam-mini: Use Fewer Learning Rates To Gain More","created":"2024-06-24","cited_by_count":137,"area":null},{"id":"2208.11625","title":"PromptFL: Let Federated Participants Cooperatively Learn Prompts Instead of Models -- Federated Learning in Age of Foundation Model","created":"2022-08-24","cited_by_count":134,"area":null},{"id":"2306.09782","title":"Full Parameter Fine-tuning for Large Language Models with Limited Resources","created":"2023-06-16","cited_by_count":134,"area":null},{"id":"2407.18921","title":"Mobile Edge Intelligence for Large Language Models: A Contemporary Survey","created":"2024-07-09","cited_by_count":128,"area":null},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126,"area":null},{"id":"2110.14378","title":"Towards artificial general intelligence via a multimodal foundation model","created":"2021-10-27","cited_by_count":112,"area":null},{"id":"2206.09557","title":"LUT-GEMM: Quantized Matrix Multiplication based on LUTs for Efficient Inference in Large-Scale Generative Language Models","created":"2022-06-20","cited_by_count":105,"area":null},{"id":"2209.01188","title":"Petals: Collaborative Inference and Fine-tuning of Large Models","created":"2022-09-02","cited_by_count":87,"area":null},{"id":"2203.12533","title":"Pathways: Asynchronous Distributed Dataflow for ML","created":"2022-03-23","cited_by_count":82,"area":null},{"id":"2309.06497","title":"A Distributed Data-Parallel PyTorch Implementation of the Distributed Shampoo Optimizer for Training Neural Networks At-Scale","created":"2023-09-12","cited_by_count":77,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2401.02038","title":"Understanding LLMs: A Comprehensive Overview from Training to Inference","created":"2024-01-04","cited_by_count":59,"area":null},{"id":"2406.06858","title":"FLUX: Fast Software-based Communication Overlap On GPUs Through Kernel Fusion","created":"2024-06-11","cited_by_count":57,"area":null},{"id":"2309.01507","title":"Memory Efficient Optimizers with 4-bit States","created":"2023-09-04","cited_by_count":54,"area":null},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52,"area":null},{"id":"2303.14006","title":"ASTRA-sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-model Training at Scale","created":"2023-03-24","cited_by_count":47,"area":null},{"id":"2403.07648","title":"Characterization of Large Language Model Development in the Datacenter","created":"2024-03-12","cited_by_count":46,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2309.08125","title":"Oobleck: Resilient Distributed Training of Large Models Using Pipeline Templates","created":"2023-09-15","cited_by_count":40,"area":null},{"id":"2107.08147","title":"AutoFL: Enabling Heterogeneity-Aware Energy Efficient Federated Learning","created":"2021-07-16","cited_by_count":39,"area":null},{"id":"2207.02852","title":"Machine Learning Model Sizes and the Parameter Gap","created":"2022-07-05","cited_by_count":39,"area":null},{"id":"2402.15043","title":"KIEval: A Knowledge-grounded Interactive Evaluation Framework for Large Language Models","created":"2024-02-23","cited_by_count":36,"area":null},{"id":"2201.07705","title":"GEMEL: Model Merging for Memory-Efficient, Real-Time Video Analytics at the Edge","created":"2022-01-19","cited_by_count":32,"area":null},{"id":"2110.03888","title":"M6-10T: A Sharing-Delinking Paradigm for Efficient Multi-Trillion Parameter Pretraining","created":"2021-10-08","cited_by_count":31,"area":null},{"id":"2404.02827","title":"BAdam: A Memory Efficient Full Parameter Optimization Method for Large Language Models","created":"2024-04-03","cited_by_count":27,"area":null},{"id":"2304.03589","title":"On Efficient Training of Large-Scale Deep Learning Models: A Literature Review","created":"2023-04-07","cited_by_count":26,"area":null},{"id":"2211.14133","title":"PipeFisher: Efficient Training of Large Language Models Using Pipelining and Fisher Information Matrices","created":"2022-11-25","cited_by_count":24,"area":null},{"id":"2105.14450","title":"Maximizing Parallelism in Distributed Training for Huge Neural Networks","created":"2021-05-30","cited_by_count":23,"area":null},{"id":"2312.12391","title":"vTrain: A Simulation Framework for Evaluating Cost-effective and Compute-optimal Large Language Model Training","created":"2023-11-27","cited_by_count":23,"area":null},{"id":"2410.07265","title":"A Survey: Collaborative Hardware and Software Design in the Era of Large Language Models","created":"2024-10-08","cited_by_count":23,"area":null},{"id":"2105.14500","title":"Tesseract: Parallelize the Tensor Parallelism Efficiently","created":"2021-05-30","cited_by_count":19,"area":null},{"id":"2211.05322","title":"On Optimizing the Communication of Model Parallelism","created":"2022-11-10","cited_by_count":19,"area":null},{"id":"2110.08460","title":"A Short Study on Compressing Decoder-Based Language Models","created":"2021-10-16","cited_by_count":16,"area":null},{"id":"2401.16677","title":"T3: Transparent Tracking & Triggering for Fine-grained Overlap of Compute & Collectives","created":"2024-01-30","cited_by_count":16,"area":null},{"id":"2406.10707","title":"DataStates-LLM: Lazy Asynchronous Checkpointing for Large Language Models","created":"2024-06-15","cited_by_count":16,"area":null},{"id":"2108.05818","title":"PatrickStar: Parallel Training of Pre-trained Models via Chunk-based Memory Management","created":"2021-08-12","cited_by_count":15,"area":null},{"id":"2110.13005","title":"AxoNN: An asynchronous, message-driven parallel framework for extreme-scale deep learning","created":"2021-10-25","cited_by_count":15,"area":null},{"id":"2110.15032","title":"OneFlow: Redesign the Distributed Deep Learning Framework from Scratch","created":"2021-10-28","cited_by_count":15,"area":null},{"id":"2301.09830","title":"Optimus-CC: Efficient Large NLP Model Training with 3D Parallelism Aware Communication Compression","created":"2023-01-24","cited_by_count":15,"area":null},{"id":"2304.08480","title":"DisCo-CLIP: A Distributed Contrastive Loss for Memory Efficient CLIP Training","created":"2023-04-17","cited_by_count":15,"area":null},{"id":"2403.06664","title":"Smart-Infinity: Fast Large Language Model Training using Near-Storage Processing on a Real System","created":"2024-03-11","cited_by_count":15,"area":null},{"id":"2203.00854","title":"FastFold: Reducing AlphaFold Training Time from 11 Days to 67 Hours","created":"2022-03-02","cited_by_count":14,"area":null},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14,"area":null},{"id":"2212.04129","title":"Deep Incubation: Training Large Models by Divide-and-Conquering","created":"2022-12-08","cited_by_count":13,"area":null},{"id":"2403.12900","title":"Toward Sustainable GenAI using Generation Directives for Carbon-Friendly Large Language Model Inference","created":"2024-03-19","cited_by_count":13,"area":null},{"id":"2507.12549","title":"The Serial Scaling Hypothesis","created":"2025-07-16","cited_by_count":13,"area":null},{"id":"2203.11014","title":"DHEN: A Deep and Hierarchical Ensemble Network for Large-Scale Click-Through Rate Prediction","created":"2022-03-11","cited_by_count":12,"area":null},{"id":"2311.03687","title":"Dissecting the Runtime Performance of the Training, Fine-tuning, and Inference of Large Language Models","created":"2023-11-07","cited_by_count":12,"area":null},{"id":"2401.15207","title":"HiFT: A Hierarchical Full Parameter Fine-Tuning Strategy","created":"2024-01-26","cited_by_count":12,"area":null},{"id":"2405.20835","title":"Outliers and Calibration Sets have Diminishing Effect on Quantization of Modern LLMs","created":"2024-05-31","cited_by_count":12,"area":null},{"id":"2111.05972","title":"Amazon SageMaker Model Parallelism: A General and Flexible Framework for Large Model Training","created":"2021-11-10","cited_by_count":11,"area":null},{"id":"2205.04702","title":"Training Personalized Recommendation Systems from (GPU) Scratch: Look Forward not Backwards","created":"2022-05-10","cited_by_count":11,"area":null},{"id":"2206.04959","title":"Merak: An Efficient Distributed DNN Training Framework with Automated 3D Parallelism for Giant Foundation Models","created":"2022-06-10","cited_by_count":11,"area":null},{"id":"2401.02643","title":"Training and Serving System of Foundation Models: A Comprehensive Survey","created":"2024-01-05","cited_by_count":11,"area":null},{"id":"2208.08263","title":"Multimodal foundation models are better simulators of the human brain","created":"2022-08-17","cited_by_count":10,"area":null},{"id":"2310.09443","title":"G10: Enabling An Efficient Unified GPU Memory and Storage Architecture with Smart Tensor Migrations","created":"2023-10-13","cited_by_count":10,"area":null},{"id":"2402.01158","title":"LLM-Detector: Improving AI-Generated Chinese Text Detection with Open-Source LLM Instruction Tuning","created":"2024-02-02","cited_by_count":10,"area":null},{"id":"2110.08554","title":"PAGnol: An Extra-Large French Generative Model","created":"2021-10-16","cited_by_count":9,"area":null},{"id":"2205.13722","title":"Can Foundation Models Help Us Achieve Perfect Secrecy?","created":"2022-05-27","cited_by_count":9,"area":null},{"id":"2510.25771","title":"Gaperon: A Peppered English-French Generative Language Model Suite","created":"2025-10-29","cited_by_count":9,"area":null},{"id":"2303.02868","title":"Angel-PTM: A Scalable and Economical Large-scale Pre-training System in Tencent","created":"2023-03-06","cited_by_count":8,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2111.04949","title":"A Survey and Empirical Evaluation of Parallel Deep Learning Frameworks","created":"2021-11-09","cited_by_count":7,"area":null},{"id":"2311.11822","title":"Zero redundancy distributed learning with differential privacy","created":"2023-11-20","cited_by_count":7,"area":null},{"id":"2404.06003","title":"FreeEval: A Modular Framework for Trustworthy and Efficient Evaluation of Large Language Models","created":"2024-04-09","cited_by_count":7,"area":null},{"id":"2409.01990","title":"Designing Large Foundation Models for Efficient Training and Inference: A Survey","created":"2024-09-03","cited_by_count":7,"area":null},{"id":"2410.21316","title":"Deep Optimizer States: Towards Scalable Training of Transformer Models Using Interleaved Offloading","created":"2024-10-26","cited_by_count":7,"area":null},{"id":"2503.19050","title":"Mist: Efficient Distributed Training of Large Language Models via Memory-Parallelism Co-Optimization","created":"2025-03-24","cited_by_count":7,"area":null},{"id":"2106.02679","title":"Layered gradient accumulation and modular pipeline parallelism: fast and efficient training of large language models","created":"2021-06-04","cited_by_count":6,"area":null},{"id":"2202.01306","title":"Harmony: Overcoming the Hurdles of GPU Memory Capacity to Train Massive DNN Models on Commodity Servers","created":"2022-02-02","cited_by_count":6,"area":null},{"id":"2206.14098","title":"RevBiFPN: The Fully Reversible Bidirectional Feature Pyramid Network","created":"2022-06-28","cited_by_count":6,"area":null},{"id":"2306.16384","title":"Accelerating Sampling and Aggregation Operations in GNN Frameworks with GPU Initiated Direct Storage Accesses","created":"2023-06-28","cited_by_count":6,"area":null},{"id":"2406.02290","title":"A Study of Optimizations for Fine-tuning Large Language Models","created":"2024-06-04","cited_by_count":6,"area":null},{"id":"2406.02613","title":"ACCO: Accumulate While You Communicate for Communication-Overlapped Sharded LLM Training","created":"2024-06-03","cited_by_count":6,"area":null},{"id":"2406.13768","title":"FastPersist: Accelerating Model Checkpointing in Deep Learning","created":"2024-06-19","cited_by_count":6,"area":null},{"id":"2407.15892","title":"Mini-Sequence Transformer: Optimizing Intermediate Memory for Long Sequences Training","created":"2024-07-22","cited_by_count":6,"area":null},{"id":"2509.21271","title":"SuperOffload: Unleashing the Power of Large-Scale LLM Training on Superchips","created":"2025-09-25","cited_by_count":6,"area":null},{"id":"2112.02752","title":"End-to-end Adaptive Distributed Training on PaddlePaddle","created":"2021-12-06","cited_by_count":5,"area":null},{"id":"2206.04688","title":"A New Frontier of AI: On-Device AI Training and Personalization","created":"2022-06-09","cited_by_count":5,"area":null},{"id":"2208.12097","title":"Training a T5 Using Lab-sized Resources","created":"2022-08-25","cited_by_count":5,"area":null},{"id":"2302.02599","title":"Colossal-Auto: Unified Automation of Parallelization and Activation Checkpoint for Large-scale Models","created":"2023-02-06","cited_by_count":5,"area":null},{"id":"2302.05045","title":"Exploiting Sparsity in Pruned Neural Networks to Optimize Large Model Training","created":"2023-02-10","cited_by_count":5,"area":null},{"id":"2305.19982","title":"Adam Accumulation to Reduce Memory Footprints of both Activations and Gradients for Large-scale DNN Training","created":"2023-05-31","cited_by_count":5,"area":null},{"id":"2311.13225","title":"NeutronOrch: Rethinking Sample-based GNN Training under CPU-GPU Heterogeneous Environments","created":"2023-11-22","cited_by_count":5,"area":null}],"next_cursor":"100"}