{"total":156,"rows":[{"id":"2106.07139","title":"Pre-Trained Models: Past, Present and Future","created":"2021-06-14","cited_by_count":319,"area":null},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256,"area":null},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245,"area":null},{"id":"2312.15234","title":"Towards Efficient Generative Large Language Model Serving: A Survey from Algorithms to Systems","created":"2023-12-23","cited_by_count":102,"area":"inference"},{"id":"2107.06925","title":"Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines","created":"2021-07-14","cited_by_count":68,"area":null},{"id":"2206.07638","title":"Asynchronous SGD Beats Minibatch SGD Under Arbitrary Delays","created":"2022-06-15","cited_by_count":63,"area":null},{"id":"2301.11913","title":"SWARM Parallelism: Training Large Models Can Be Surprisingly Communication-Efficient","created":"2023-01-27","cited_by_count":59,"area":null},{"id":"2110.13363","title":"Exponential Graph is Provably Efficient for Decentralized Deep Training","created":"2021-10-26","cited_by_count":56,"area":null},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52,"area":null},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42,"area":null},{"id":"2210.06640","title":"Compute-Efficient Deep Learning: Algorithmic Trends and Opportunities","created":"2022-10-13","cited_by_count":38,"area":null},{"id":"2511.09057","title":"PAN: A World Model for General, Actionable, and Long-Horizon World Simulation","created":"2025-11-12","cited_by_count":36,"area":"wm"},{"id":"2106.05268","title":"Vector Symbolic Architectures as a Computing Framework for Emerging Hardware","created":"2021-06-09","cited_by_count":35,"area":null},{"id":"2103.11109","title":"DataLens: Scalable Privacy Preserving Training via Gradient Compression and Aggregation","created":"2021-03-20","cited_by_count":31,"area":null},{"id":"2111.14655","title":"FedHM: Efficient Federated Learning for Heterogeneous Models via Low-rank Factorization","created":"2021-11-29","cited_by_count":31,"area":null},{"id":"2103.06995","title":"Performance of a Geometric Deep Learning Pipeline for HL-LHC Particle Tracking","created":"2021-03-11","cited_by_count":30,"area":null},{"id":"2205.09702","title":"Parallel and Distributed Graph Neural Networks: An In-Depth Concurrency Analysis","created":"2022-05-19","cited_by_count":29,"area":null},{"id":"2203.13789","title":"FLUTE: A Scalable, Extensible Framework for High-Performance Federated Learning Simulations","created":"2022-03-25","cited_by_count":27,"area":null},{"id":"2308.00852","title":"CASSINI: Network-Aware Job Scheduling in Machine Learning Clusters","created":"2023-08-01","cited_by_count":26,"area":null},{"id":"2105.09080","title":"Accelerating Gossip SGD with Periodic Global Averaging","created":"2021-05-19","cited_by_count":25,"area":null},{"id":"2201.07598","title":"Near-Optimal Sparse Allreduce for Distributed Deep Learning","created":"2022-01-19","cited_by_count":23,"area":null},{"id":"2012.15589","title":"PFL-MoE: Personalized Federated Learning Based on Mixture of Experts","created":"2020-12-31","cited_by_count":21,"area":"moe"},{"id":"2210.08803","title":"Merlin HugeCTR: GPU-accelerated Recommender System Training and Inference","created":"2022-10-17","cited_by_count":20,"area":null},{"id":"2306.17648","title":"Enhancing training of physics-informed neural networks using domain-decomposition based preconditioning strategies","created":"2023-06-30","cited_by_count":20,"area":null},{"id":"2105.13205","title":"Hamiltonian Deep Neural Networks Guaranteeing Non-vanishing Gradients by Design","created":"2021-05-27","cited_by_count":18,"area":null},{"id":"2107.00961","title":"ResIST: Layer-Wise Decomposition of ResNets for Distributed Training","created":"2021-07-02","cited_by_count":18,"area":null},{"id":"2104.05245","title":"Distributed Learning Systems with First-order Methods","created":"2021-04-12","cited_by_count":17,"area":null},{"id":"2102.01732","title":"Truly Sparse Neural Networks at Scale","created":"2021-02-02","cited_by_count":16,"area":null},{"id":"2111.04287","title":"BlueFog: Make Decentralized Algorithms Practical for Optimization and Deep Learning","created":"2021-11-08","cited_by_count":16,"area":null},{"id":"2110.15032","title":"OneFlow: Redesign the Distributed Deep Learning Framework from Scratch","created":"2021-10-28","cited_by_count":15,"area":null},{"id":"2503.06433","title":"Seesaw: High-throughput LLM Inference via Model Re-sharding","created":"2025-03-09","cited_by_count":15,"area":"inference"},{"id":"2101.08734","title":"Clairvoyant Prefetching for Distributed Machine Learning I/O","created":"2021-01-21","cited_by_count":14,"area":null},{"id":"2305.13854","title":"The Evolution of Distributed Systems for Graph Neural Networks and their Origin in Graph Processing and Deep Learning: A Survey","created":"2023-05-23","cited_by_count":14,"area":null},{"id":"2312.14050","title":"Machine learning and domain decomposition methods -- a survey","created":"2023-12-21","cited_by_count":14,"area":null},{"id":"2211.07056","title":"An Invitation to Distributed Quantum Neural Networks","created":"2022-11-14","cited_by_count":13,"area":null},{"id":"2302.02390","title":"Quantized Distributed Training of Large Models with Convergence Guarantees","created":"2023-02-05","cited_by_count":13,"area":null},{"id":"2305.12213","title":"Taming Resource Heterogeneity In Distributed ML Training With Dynamic Batching","created":"2023-05-20","cited_by_count":12,"area":null},{"id":"2312.02515","title":"mLoRA: Fine-Tuning LoRA Adapters via Highly-Efficient Pipeline Parallelism in Multiple GPUs","created":"2023-12-05","cited_by_count":12,"area":null},{"id":"2306.17281","title":"HPC-Coder: Modeling Parallel Programs using Large Language Models","created":"2023-06-29","cited_by_count":11,"area":null},{"id":"2311.00465","title":"Asynchronous SGD on Graphs: a Unified Framework for Asynchronous Decentralized and Federated Optimization","created":"2023-11-01","cited_by_count":11,"area":null},{"id":"2102.10424","title":"GIST: Distributed Training for Large-Scale Graph Convolutional Networks","created":"2021-02-20","cited_by_count":10,"area":null},{"id":"2111.00856","title":"Large-Scale Deep Learning Optimizations: A Comprehensive Survey","created":"2021-11-01","cited_by_count":10,"area":null},{"id":"2404.06114","title":"Communication-Efficient Large-Scale Distributed Deep Learning: A Comprehensive Survey","created":"2024-04-09","cited_by_count":10,"area":null},{"id":"2210.08804","title":"A GPU-specialized Inference Parameter Server for Large-Scale Deep Recommendation Models","created":"2022-10-17","cited_by_count":9,"area":null},{"id":"2309.16318","title":"DeepPCR: Parallelizing Sequential Operations in Neural Networks","created":"2023-09-28","cited_by_count":9,"area":"diffusion"},{"id":"2407.14645","title":"Performance Modeling and Workload Analysis of Distributed Large Language Model Training and Inference","created":"2024-07-19","cited_by_count":9,"area":null},{"id":"2102.07214","title":"Communication-Efficient Distributed Optimization with Quantized Preconditioners","created":"2021-02-14","cited_by_count":8,"area":null},{"id":"2111.09562","title":"COMET: A Novel Memory-Efficient Deep Learning Training Framework by Using Error-Bounded Lossy Compression","created":"2021-11-18","cited_by_count":8,"area":null},{"id":"2112.01637","title":"AdaSplit: Adaptive Trade-offs for Resource-constrained Distributed Deep Learning","created":"2021-12-02","cited_by_count":8,"area":null},{"id":"2209.01346","title":"HammingMesh: A Network Topology for Large-Scale Deep Learning","created":"2022-09-03","cited_by_count":8,"area":null},{"id":"2209.04184","title":"Anomaly Detection through Unsupervised Federated Learning","created":"2022-09-09","cited_by_count":8,"area":null},{"id":"2401.09356","title":"Swing: Short-cutting Rings for Higher Bandwidth Allreduce","created":"2024-01-17","cited_by_count":8,"area":null},{"id":"2411.05614","title":"Acceleration for Deep Reinforcement Learning using Parallel and Distributed Computing: A Survey","created":"2024-11-08","cited_by_count":8,"area":"deeprl"},{"id":"2106.12261","title":"Learning Under Delayed Feedback: Implicitly Adapting to Gradient Delays","created":"2021-06-23","cited_by_count":7,"area":null},{"id":"2111.04949","title":"A Survey and Empirical Evaluation of Parallel Deep Learning Frameworks","created":"2021-11-09","cited_by_count":7,"area":null},{"id":"2208.02442","title":"FedDRL: Deep Reinforcement Learning-based Adaptive Aggregation for Non-IID Data in Federated Learning","created":"2022-08-04","cited_by_count":7,"area":null},{"id":"2310.12802","title":"An effective theory of collective deep learning","created":"2023-10-19","cited_by_count":7,"area":null},{"id":"2408.10197","title":"Demystifying the Communication Characteristics for Distributed Transformer Models","created":"2024-08-19","cited_by_count":7,"area":null},{"id":"2106.03640","title":"Making EfficientNet More Efficient: Exploring Batch-Independent Normalization, Group Convolutions and Reduced Resolution Training","created":"2021-06-07","cited_by_count":6,"area":null},{"id":"2306.00256","title":"DSGD-CECA: Decentralized SGD with Communication-Optimal Exact Consensus Algorithm","created":"2023-06-01","cited_by_count":6,"area":null},{"id":"2102.01963","title":"Local Critic Training for Model-Parallel Learning of Deep Neural Networks","created":"2021-02-03","cited_by_count":5,"area":null},{"id":"2104.14538","title":"Distributed Multigrid Neural Solvers on Megavoxel Domains","created":"2021-04-29","cited_by_count":5,"area":null},{"id":"2106.05373","title":"StreamBrain: An HPC Framework for Brain-like Neural Networks on CPUs, GPUs and FPGAs","created":"2021-06-09","cited_by_count":5,"area":null},{"id":"2203.04738","title":"Parallel Training of GRU Networks with a Multi-Grid Solver for Long Sequences","created":"2022-03-07","cited_by_count":5,"area":null},{"id":"2302.05045","title":"Exploiting Sparsity in Pruned Neural Networks to Optimize Large Model Training","created":"2023-02-10","cited_by_count":5,"area":null},{"id":"2302.13562","title":"Communication-efficient Federated Learning with Single-Step Synthetic Features Compressor for Faster Convergence","created":"2023-02-27","cited_by_count":5,"area":null},{"id":"2303.06659","title":"Scavenger: A Cloud Service for Optimizing Cost and Performance of ML Training","created":"2023-03-12","cited_by_count":5,"area":null},{"id":"2406.10997","title":"Two-level overlapping additive Schwarz preconditioner for training scientific machine learning applications","created":"2024-06-16","cited_by_count":5,"area":null},{"id":"2601.12554","title":"Artificial Intelligence in Materials Science and Engineering: Current Landscape, Key Challenges, and Future Trajectorie","created":"2026-01-18","cited_by_count":5,"area":null},{"id":"2101.09001","title":"Linear Regression with Distributed Learning: A Generalization Error Perspective","created":"2021-01-22","cited_by_count":4,"area":null},{"id":"2103.08802","title":"Parareal Neural Networks Emulating a Parallel-in-time Algorithm","created":"2021-03-16","cited_by_count":4,"area":null},{"id":"2104.05588","title":"Accelerating Neural Network Training with Distributed Asynchronous and Selective Optimization (DASO)","created":"2021-04-12","cited_by_count":4,"area":null},{"id":"2106.07578","title":"Dynamic Gradient Aggregation for Federated Domain Adaptation","created":"2021-06-14","cited_by_count":4,"area":null},{"id":"2110.08688","title":"MG-GCN: Scalable Multi-GPU GCN Training Framework","created":"2021-10-17","cited_by_count":4,"area":null},{"id":"2212.00253","title":"Distributed Deep Reinforcement Learning: A Survey and A Multi-Player Multi-Agent Learning Toolbox","created":"2022-12-01","cited_by_count":4,"area":null},{"id":"2306.10598","title":"DropCompute: simple and more robust distributed synchronous training via compute variance reduction","created":"2023-06-18","cited_by_count":4,"area":null},{"id":"2312.00839","title":"PipeOptim: Ensuring Effective 1F1B Schedule with Optimizer-Dependent Weight Prediction","created":"2023-12-01","cited_by_count":4,"area":null},{"id":"2402.07529","title":"Accelerating Distributed Deep Learning using Lossless Homomorphic Compression","created":"2024-02-12","cited_by_count":4,"area":null},{"id":"2403.04206","title":"GRAWA: Gradient-based Weighted Averaging for Distributed Training of Deep Learning Models","created":"2024-03-07","cited_by_count":4,"area":null},{"id":"2410.05930","title":"Fortify Your Foundations: Practical Privacy and Security for Foundation Model Deployments In The Cloud","created":"2024-10-08","cited_by_count":4,"area":null},{"id":"2104.08364","title":"Sync-Switch: Hybrid Parameter Synchronization for Distributed Deep Learning","created":"2021-04-16","cited_by_count":3,"area":null},{"id":"2106.03585","title":"Asynchronous speedup in decentralized optimization","created":"2021-06-07","cited_by_count":3,"area":null},{"id":"2106.12091","title":"BFTrainer: Low-Cost Training of Neural Networks on Unfillable Supercomputer Nodes","created":"2021-06-22","cited_by_count":3,"area":null},{"id":"2110.08633","title":"Hydra: A System for Large Multi-Model Deep Learning","created":"2021-10-16","cited_by_count":3,"area":null},{"id":"2110.12484","title":"Enabling Large Batch Size Training for DNN Models Beyond the Memory Limit While Maintaining Performance","created":"2021-10-24","cited_by_count":3,"area":null},{"id":"2112.10065","title":"Efficient Strong Scaling Through Burst Parallel Training","created":"2021-12-19","cited_by_count":3,"area":null},{"id":"2210.16169","title":"LOFT: Finding Lottery Tickets through Filter-wise Training","created":"2022-10-28","cited_by_count":3,"area":null},{"id":"2302.12445","title":"DeAR: Accelerating Distributed Deep Learning with Fine-Grained All-Reduce Pipelining","created":"2023-02-24","cited_by_count":3,"area":null},{"id":"2502.03092","title":"E-3SFC: Communication-Efficient Federated Learning with Double-way Features Synthesizing","created":"2025-02-05","cited_by_count":3,"area":null},{"id":"2502.19790","title":"Mixtera: A Data Plane for Foundation Model Training","created":"2025-02-27","cited_by_count":3,"area":null},{"id":"2503.23830","title":"OrchMLLM: Orchestrate Multimodal Data with Batch Post-Balancing to Accelerate Multimodal Large Language Model Training","created":"2025-03-31","cited_by_count":3,"area":"mllm"},{"id":"2104.09075","title":"An Oracle for Guiding Large-Scale Model/Hybrid Parallel Training of Convolutional Neural Networks","created":"2021-04-19","cited_by_count":2,"area":null},{"id":"2106.09831","title":"On Effects of Compression with Hyperdimensional Computing in Distributed Randomized Neural Networks","created":"2021-06-17","cited_by_count":2,"area":null},{"id":"2108.06629","title":"LayerPipe: Accelerating Deep Neural Network Training by Intra-Layer and Inter-Layer Gradient Pipelining and Multiprocessor Scheduling","created":"2021-08-14","cited_by_count":2,"area":null},{"id":"2211.12709","title":"SciAI4Industry -- Solving PDEs for industry-scale problems with deep learning","created":"2022-11-23","cited_by_count":2,"area":null},{"id":"2309.03754","title":"Convergence Analysis of Decentralized ASGD","created":"2023-09-07","cited_by_count":2,"area":null},{"id":"2310.19503","title":"Trust, Accountability, and Autonomy in Knowledge Graph-based AI for Self-determination","created":"2023-10-30","cited_by_count":2,"area":null},{"id":"2312.13677","title":"Parallel Trust-Region Approaches in Neural Network Training: Beyond Traditional Methods","created":"2023-12-21","cited_by_count":2,"area":null},{"id":"2406.17812","title":"Scalable Artificial Intelligence for Science: Perspectives, Methods and Exemplars","created":"2024-06-24","cited_by_count":2,"area":null},{"id":"2408.14442","title":"Model Parallel Training and Transfer Learning for Convolutional Neural Networks by Domain Decomposition","created":"2024-08-26","cited_by_count":2,"area":null}],"next_cursor":"100"}