{"total":260,"rows":[{"id":"2204.02311","title":"PaLM: Scaling Language Modeling with Pathways","created":"2022-04-05","cited_by_count":5326,"area":null},{"id":"2408.00118","title":"Gemma 2: Improving Open Language Models at a Practical Size","created":"2024-07-31","cited_by_count":1998,"area":null},{"id":"2503.19786","title":"Gemma 3 Technical Report","created":"2025-03-25","cited_by_count":1824,"area":null},{"id":"2403.13372","title":"LlamaFactory: Unified Efficient Fine-Tuning of 100+ Language Models","created":"2024-03-20","cited_by_count":1464,"area":null},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660,"area":null},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632,"area":null},{"id":"2303.06865","title":"FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU","created":"2023-03-13","cited_by_count":575,"area":null},{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2306.05087","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","created":"2023-06-08","cited_by_count":316,"area":null},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2308.00708","title":"VeriGen: A Large Language Model for Verilog Code Generation","created":"2023-07-28","cited_by_count":200,"area":null},{"id":"2312.03863","title":"Efficient Large Language Models: A Survey","created":"2023-12-06","cited_by_count":186,"area":null},{"id":"2309.14322","title":"Small-scale proxies for large-scale Transformer training instabilities","created":"2023-09-25","cited_by_count":183,"area":null},{"id":"2310.05470","title":"Generative Judge for Evaluating Alignment","created":"2023-10-09","cited_by_count":169,"area":null},{"id":"2310.05694","title":"A Survey of Large Language Models for Healthcare: from Data, Technology, and Applications to Accountability and Ethics","created":"2023-10-09","cited_by_count":163,"area":null},{"id":"2309.00363","title":"FederatedScope-LLM: A Comprehensive Package for Fine-tuning Large Language Models in Federated Learning","created":"2023-09-01","cited_by_count":149,"area":null},{"id":"2306.09782","title":"Full Parameter Fine-tuning for Large Language Models with Limited Resources","created":"2023-06-16","cited_by_count":134,"area":null},{"id":"2407.18921","title":"Mobile Edge Intelligence for Large Language Models: A Contemporary Survey","created":"2024-07-09","cited_by_count":128,"area":null},{"id":"2105.13120","title":"Sequence Parallelism: Long Sequence Training from System Perspective","created":"2021-05-26","cited_by_count":126,"area":null},{"id":"2402.11592","title":"Revisiting Zeroth-Order Optimization for Memory-Efficient LLM Fine-Tuning: A Benchmark","created":"2024-02-18","cited_by_count":114,"area":null},{"id":"2308.08239","title":"MemoChat: Tuning LLMs to Use Memos for Consistent Long-Range Open-Domain Conversation","created":"2023-08-16","cited_by_count":97,"area":null},{"id":"2110.14883","title":"Colossal-AI: A Unified Deep Learning System For Large-Scale Parallel Training","created":"2021-10-28","cited_by_count":96,"area":null},{"id":"2311.12023","title":"LQ-LoRA: Low-rank Plus Quantized Matrix Decomposition for Efficient Language Model Finetuning","created":"2023-11-20","cited_by_count":92,"area":null},{"id":"2403.17919","title":"LISA: Layerwise Importance Sampling for Memory-Efficient Large Language Model Fine-Tuning","created":"2024-03-26","cited_by_count":89,"area":null},{"id":"2206.01288","title":"Decentralized Training of Foundation Models in Heterogeneous Environments","created":"2022-06-02","cited_by_count":87,"area":null},{"id":"2209.01188","title":"Petals: Collaborative Inference and Fine-tuning of Large Models","created":"2022-09-02","cited_by_count":87,"area":null},{"id":"2401.08092","title":"A Survey of Resource-efficient LLM and Multimodal Foundation Models","created":"2024-01-16","cited_by_count":86,"area":null},{"id":"2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","created":"2022-12-28","cited_by_count":81,"area":null},{"id":"2209.00099","title":"Efficient Methods for Natural Language Processing: A Survey","created":"2022-08-31","cited_by_count":78,"area":null},{"id":"2102.07988","title":"TeraPipe: Token-Level Pipeline Parallelism for Training Large-Scale Language Models","created":"2021-02-16","cited_by_count":75,"area":null},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68,"area":null},{"id":"2205.00049","title":"Prompt Consistency for Zero-Shot Task Generalization","created":"2022-04-29","cited_by_count":67,"area":null},{"id":"2104.14129","title":"ActNN: Reducing Training Memory Footprint via 2-Bit Activation Compressed Training","created":"2021-04-29","cited_by_count":61,"area":null},{"id":"2405.07719","title":"USP: A Unified Sequence Parallelism Approach for Long Context Generative AI","created":"2024-05-13","cited_by_count":61,"area":null},{"id":"2405.14852","title":"PV-Tuning: Beyond Straight-Through Estimation for Extreme LLM Compression","created":"2024-05-23","cited_by_count":61,"area":null},{"id":"2306.12420","title":"LMFlow: An Extensible Toolkit for Finetuning and Inference of Large Foundation Models","created":"2023-06-21","cited_by_count":60,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2401.02038","title":"Understanding LLMs: A Comprehensive Overview from Training to Inference","created":"2024-01-04","cited_by_count":59,"area":null},{"id":"2106.10207","title":"Distributed Deep Learning in Open Collaborations","created":"2021-06-18","cited_by_count":58,"area":null},{"id":"2406.06858","title":"FLUX: Fast Software-based Communication Overlap On GPUs Through Kernel Fusion","created":"2024-06-11","cited_by_count":57,"area":null},{"id":"2303.10845","title":"PanGu-{\\Sigma}: Towards Trillion Parameter Language Model with Sparse Heterogeneous Computing","created":"2023-03-20","cited_by_count":56,"area":null},{"id":"2309.01507","title":"Memory Efficient Optimizers with 4-bit States","created":"2023-09-04","cited_by_count":54,"area":null},{"id":"2303.14006","title":"ASTRA-sim2.0: Modeling Hierarchical Networks and Disaggregated Systems for Large-model Training at Scale","created":"2023-03-24","cited_by_count":47,"area":null},{"id":"2403.07648","title":"Characterization of Large Language Model Development in the Datacenter","created":"2024-03-12","cited_by_count":46,"area":null},{"id":"2410.01623","title":"Fira: Can We Achieve Full-rank Training of LLMs Under Low-rank Constraint?","created":"2024-10-02","cited_by_count":43,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2112.15283","title":"ERNIE-ViLG: Unified Generative Pre-training for Bidirectional Vision-Language Generation","created":"2021-12-31","cited_by_count":41,"area":null},{"id":"2309.08125","title":"Oobleck: Resilient Distributed Training of Large Models Using Pipeline Templates","created":"2023-09-15","cited_by_count":40,"area":null},{"id":"2302.01107","title":"A Survey on Efficient Training of Transformers","created":"2023-02-02","cited_by_count":39,"area":null},{"id":"2210.06640","title":"Compute-Efficient Deep Learning: Algorithmic Trends and Opportunities","created":"2022-10-13","cited_by_count":38,"area":null},{"id":"2110.03888","title":"M6-10T: A Sharing-Delinking Paradigm for Efficient Multi-Trillion Parameter Pretraining","created":"2021-10-08","cited_by_count":31,"area":null},{"id":"2206.11357","title":"GACT: Activation Compressed Training for Generic Network Architectures","created":"2022-06-22","cited_by_count":31,"area":null},{"id":"2405.10853","title":"The Future of Large Language Model Pre-training is Federated","created":"2024-05-17","cited_by_count":30,"area":null},{"id":"2403.00071","title":"Resonance RoPE: Improving Context Length Generalization of Large Language Models","created":"2024-02-29","cited_by_count":27,"area":null},{"id":"2404.02827","title":"BAdam: A Memory Efficient Full Parameter Optimization Method for Large Language Models","created":"2024-04-03","cited_by_count":27,"area":null},{"id":"2503.09799","title":"Communication-Efficient Language Model Training Scales Reliably and Robustly: Scaling Laws for DiLoCo","created":"2025-03-12","cited_by_count":27,"area":null},{"id":"2207.07697","title":"POET: Training Neural Networks on Tiny Devices with Integrated Rematerialization and Paging","created":"2022-07-15","cited_by_count":26,"area":null},{"id":"2304.03589","title":"On Efficient Training of Large-Scale Deep Learning Models: A Literature Review","created":"2023-04-07","cited_by_count":26,"area":null},{"id":"2306.00477","title":"Make Pre-trained Model Reversible: From Parameter to Memory Efficient Fine-Tuning","created":"2023-06-01","cited_by_count":26,"area":null},{"id":"2312.00678","title":"The Efficiency Spectrum of Large Language Models: An Algorithmic Survey","created":"2023-12-01","cited_by_count":26,"area":null},{"id":"2406.00770","title":"Automatic Instruction Evolving for Large Language Models","created":"2024-06-02","cited_by_count":26,"area":null},{"id":"2310.09089","title":"Qilin-Med: Multi-stage Knowledge Injection Advanced Medical Large Language Model","created":"2023-10-13","cited_by_count":24,"area":null},{"id":"2312.12391","title":"vTrain: A Simulation Framework for Evaluating Cost-effective and Compute-optimal Large Language Model Training","created":"2023-11-27","cited_by_count":23,"area":null},{"id":"2410.07265","title":"A Survey: Collaborative Hardware and Software Design in the Era of Large Language Models","created":"2024-10-08","cited_by_count":23,"area":null},{"id":"2401.07004","title":"Extending LLMs' Context Window with 100 Samples","created":"2024-01-13","cited_by_count":21,"area":null},{"id":"2201.12023","title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning","created":"2022-01-28","cited_by_count":20,"area":null},{"id":"2407.01031","title":"PocketLLM: Enabling On-Device Fine-Tuning for Personalized LLMs","created":"2024-07-01","cited_by_count":18,"area":null},{"id":"2310.03150","title":"Federated Fine-Tuning of LLMs on the Very Edge: The Good, the Bad, the Ugly","created":"2023-10-04","cited_by_count":17,"area":null},{"id":"2503.17924","title":"WLB-LLM: Workload-Balanced 4D Parallelism for Large Language Model Training","created":"2025-03-23","cited_by_count":17,"area":null},{"id":"2110.08460","title":"A Short Study on Compressing Decoder-Based Language Models","created":"2021-10-16","cited_by_count":16,"area":null},{"id":"2509.16293","title":"Robust LLM Training Infrastructure at ByteDance","created":"2025-09-19","cited_by_count":16,"area":null},{"id":"2108.05818","title":"PatrickStar: Parallel Training of Pre-trained Models via Chunk-based Memory Management","created":"2021-08-12","cited_by_count":15,"area":null},{"id":"2110.13005","title":"AxoNN: An asynchronous, message-driven parallel framework for extreme-scale deep learning","created":"2021-10-25","cited_by_count":15,"area":null},{"id":"2301.09830","title":"Optimus-CC: Efficient Large NLP Model Training with 3D Parallelism Aware Communication Compression","created":"2023-01-24","cited_by_count":15,"area":null},{"id":"2403.06664","title":"Smart-Infinity: Fast Large Language Model Training using Near-Storage Processing on a Real System","created":"2024-03-11","cited_by_count":15,"area":null},{"id":"2405.14009","title":"ReCycle: Resilient Training of Large DNNs using Pipeline Adaptation","created":"2024-05-22","cited_by_count":14,"area":null},{"id":"2407.12943","title":"Halu-J: Critique-Based Hallucination Judge","created":"2024-07-17","cited_by_count":14,"area":null},{"id":"2302.02390","title":"Quantized Distributed Training of Large Models with Convergence Guarantees","created":"2023-02-05","cited_by_count":13,"area":null},{"id":"2310.10046","title":"TRANSOM: An Efficient Fault-Tolerant System for Training LLMs","created":"2023-10-16","cited_by_count":13,"area":null},{"id":"2112.02958","title":"Automap: Towards Ergonomic Automated Parallelism for ML Models","created":"2021-12-06","cited_by_count":12,"area":null},{"id":"2311.03687","title":"Dissecting the Runtime Performance of the Training, Fine-tuning, and Inference of Large Language Models","created":"2023-11-07","cited_by_count":12,"area":null},{"id":"2312.02515","title":"mLoRA: Fine-Tuning LoRA Adapters via Highly-Efficient Pipeline Parallelism in Multiple GPUs","created":"2023-12-05","cited_by_count":12,"area":null},{"id":"2404.10933","title":"LLMem: Estimating GPU Memory Usage for Fine-Tuning Pre-Trained LLMs","created":"2024-04-16","cited_by_count":12,"area":null},{"id":"2111.05972","title":"Amazon SageMaker Model Parallelism: A General and Flexible Framework for Large Model Training","created":"2021-11-10","cited_by_count":11,"area":null},{"id":"2206.04959","title":"Merak: An Efficient Distributed DNN Training Framework with Automated 3D Parallelism for Giant Foundation Models","created":"2022-06-10","cited_by_count":11,"area":null},{"id":"2306.17281","title":"HPC-Coder: Modeling Parallel Programs using Large Language Models","created":"2023-06-29","cited_by_count":11,"area":null},{"id":"2401.02643","title":"Training and Serving System of Foundation Models: A Comprehensive Survey","created":"2024-01-05","cited_by_count":11,"area":null},{"id":"2108.11976","title":"JUWELS Booster -- A Supercomputer for Large-Scale AI Research","created":"2021-06-30","cited_by_count":10,"area":null},{"id":"2310.09443","title":"G10: Enabling An Efficient Unified GPU Memory and Storage Architecture with Smart Tensor Migrations","created":"2023-10-13","cited_by_count":10,"area":null},{"id":"2401.11202","title":"PartIR: Composing SPMD Partitioning Strategies for Machine Learning","created":"2024-01-20","cited_by_count":10,"area":null},{"id":"2403.09347","title":"BurstAttention: An Efficient Distributed Attention Framework for Extremely Long Sequences","created":"2024-03-14","cited_by_count":10,"area":null},{"id":"2205.13722","title":"Can Foundation Models Help Us Achieve Perfect Secrecy?","created":"2022-05-27","cited_by_count":9,"area":null},{"id":"2309.00378","title":"Long-Term Ad Memorability: Understanding & Generating Memorable Ads","created":"2023-09-01","cited_by_count":9,"area":null},{"id":"2211.00609","title":"A Simple, Yet Effective Approach to Finding Biases in Code Generation","created":"2022-10-31","cited_by_count":8,"area":null},{"id":"2303.02868","title":"Angel-PTM: A Scalable and Economical Large-scale Pre-training System in Tencent","created":"2023-03-06","cited_by_count":8,"area":null},{"id":"2410.11772","title":"Layer-wise Importance Matters: Less Memory for Better Performance in Parameter-efficient Fine-tuning of Large Language Models","created":"2024-10-15","cited_by_count":8,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2412.00071","title":"COAP: Memory-Efficient Training with Correlation-Aware Gradient Projection","created":"2024-11-26","cited_by_count":8,"area":null},{"id":"2502.06244","title":"PiKE: Adaptive Data Mixing for Large-Scale Multi-Task Learning Under Low Gradient Conflicts","created":"2025-02-10","cited_by_count":8,"area":null}],"next_cursor":"100"}