{"total":121,"rows":[{"id":"2403.08763","title":"Simple and Scalable Strategies to Continually Pre-train Large Language Models","created":"2024-03-13","cited_by_count":127,"area":null},{"id":"2403.07816","title":"Branch-Train-MiX: Mixing Expert LLMs into a Mixture-of-Experts LLM","created":"2024-03-12","cited_by_count":109,"area":"lr"},{"id":"2504.05295","title":"Dion: Distributed Orthonormalized Updates","created":"2025-04-07","cited_by_count":68,"area":null},{"id":"2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","created":"2024-09-05","cited_by_count":54,"area":null},{"id":"2406.16768","title":"WARP: On the Benefits of Weight Averaged Rewarded Policies","created":"2024-06-24","cited_by_count":45,"area":"alignment"},{"id":"2510.16981","title":"MuonBP: Faster Muon via Block-Periodic Orthogonalization","created":"2025-10-19","cited_by_count":35,"area":null},{"id":"2405.10853","title":"The Future of Large Language Model Pre-training is Federated","created":"2024-05-17","cited_by_count":30,"area":null},{"id":"2410.21680","title":"Revisiting Reliability in Large-Scale Machine Learning Research Clusters","created":"2024-10-29","cited_by_count":28,"area":null},{"id":"2501.18512","title":"Streaming DiLoCo with overlapping communication: Towards a Distributed Free Lunch","created":"2025-01-30","cited_by_count":28,"area":null},{"id":"2503.09799","title":"Communication-Efficient Language Model Training Scales Reliably and Robustly: Scaling Laws for DiLoCo","created":"2025-03-12","cited_by_count":27,"area":null},{"id":"2401.09135","title":"Asynchronous Local-SGD Training for Language Modeling","created":"2024-01-17","cited_by_count":26,"area":null},{"id":"2505.23725","title":"MuLoCo: Muon is a practical inner optimizer for DiLoCo","created":"2025-05-29","cited_by_count":18,"area":null},{"id":"2402.16828","title":"Training Neural Networks from Scratch with Parallel Low-Rank Adapters","created":"2024-02-26","cited_by_count":17,"area":null},{"id":"2501.05450","title":"Decentralized Diffusion Models","created":"2025-01-09","cited_by_count":13,"area":"diffusion"},{"id":"2506.01260","title":"Subspace Networks: Scaling Decentralized Training with Communication-Efficient Model Parallelism","created":"2025-06-02","cited_by_count":12,"area":null},{"id":"2508.15706","title":"Overcoming the Communication-Performance Tradeoff in LLM Pretraining","created":"2025-08-21","cited_by_count":12,"area":null},{"id":"2411.19870","title":"DeMo: Decoupled Momentum Optimization","created":"2024-11-29","cited_by_count":11,"area":null},{"id":"2502.12996","title":"Eager Updates For Overlapped Communication and Computation in DiLoCo","created":"2025-02-18","cited_by_count":11,"area":null},{"id":"2509.23106","title":"Effective Quantization of Muon Optimizer States","created":"2025-09-27","cited_by_count":11,"area":null},{"id":"2403.07128","title":"DrJAX: Scalable and Differentiable MapReduce Primitives in JAX","created":"2024-03-11","cited_by_count":9,"area":null},{"id":"2509.10439","title":"Understanding Outer Optimizers in Local SGD: Learning Rates, Momentum, and Acceleration","created":"2025-09-12","cited_by_count":9,"area":null},{"id":"2411.13055","title":"Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training","created":"2024-11-20","cited_by_count":8,"area":null},{"id":"2510.15830","title":"SNOO: Step-K Nesterov Outer Optimizer - The Surprising Effectiveness of Nesterov Momentum Applied to Pseudo-Gradients","created":"2025-10-17","cited_by_count":8,"area":null},{"id":"2601.21972","title":"Learning Decentralized LLM Collaboration with Multi-Agent Actor Critic","created":"2026-01-29","cited_by_count":8,"area":"agents"},{"id":"2502.04164","title":"Efficient Distributed Optimization under Heavy-Tailed Noise","created":"2025-02-06","cited_by_count":7,"area":null},{"id":"2602.03839","title":"Understanding and Exploiting Weight Update Sparsity for Communication-Efficient Distributed RL","created":"2026-02-03","cited_by_count":7,"area":null},{"id":"2605.19095","title":"ScheduleFree+: Scaling Learning-Rate-Free & Schedule-Free Learning to Large Language Models","created":"2026-05-18","cited_by_count":7,"area":null},{"id":"2406.02613","title":"ACCO: Accumulate While You Communicate for Communication-Overlapped Sharded LLM Training","created":"2024-06-03","cited_by_count":6,"area":null},{"id":"2503.09956","title":"DeepSeek-Inspired Exploration of RL-based LLMs and Synergy with Wireless Networks: A Survey","created":"2025-03-13","cited_by_count":6,"area":null},{"id":"2603.08163","title":"Covenant-72B: Pre-Training a 72B LLM with Trustless Peers Over-the-Internet","created":"2026-03-09","cited_by_count":6,"area":null},{"id":"2405.14446","title":"Worldwide Federated Training of Language Models","created":"2024-05-23","cited_by_count":5,"area":null},{"id":"2410.03529","title":"No Need to Talk: Asynchronous Mixture of Language Models","created":"2024-10-04","cited_by_count":5,"area":null},{"id":"2410.05021","title":"DEPT: Decoupled Embeddings for Pre-training Language Models","created":"2024-10-07","cited_by_count":5,"area":null},{"id":"2502.17340","title":"Low-rank bias, weight decay, and model merging in neural networks","created":"2025-02-24","cited_by_count":5,"area":null},{"id":"2506.10911","title":"NoLoCo: No-all-reduce Low Communication Training Method for Large Models","created":"2025-06-12","cited_by_count":5,"area":null},{"id":"2512.17131","title":"Smoothing DiLoCo with Primal Averaging for Faster Training of LLMs","created":"2025-12-18","cited_by_count":5,"area":null},{"id":"2406.13936","title":"Communication-Efficient Adaptive Batch Size Strategies for Distributed Local Gradient Methods","created":"2024-06-20","cited_by_count":4,"area":null},{"id":"2502.19385","title":"HDEE: Heterogeneous Domain Expert Ensemble","created":"2025-02-26","cited_by_count":4,"area":null},{"id":"2502.19913","title":"SkipPipe: Partial and Reordered Pipelining Framework for Training LLMs in Heterogeneous Networks","created":"2025-02-27","cited_by_count":4,"area":null},{"id":"2505.22549","title":"DES-LOC: Desynced Low Communication Adaptive Optimizers for Training Foundation Models","created":"2025-05-28","cited_by_count":4,"area":null},{"id":"2506.21263","title":"DiLoCoX: A Low-Communication Large-Scale Training Framework for Decentralized Cluster","created":"2025-06-26","cited_by_count":4,"area":null},{"id":"2511.19468","title":"Towards a future space-based, highly scalable AI infrastructure system design","created":"2025-11-22","cited_by_count":4,"area":null},{"id":"2603.03597","title":"NuMuon: Nuclear-Norm-Constrained Muon for Compressible LLM Training","created":"2026-03-04","cited_by_count":4,"area":null},{"id":"2604.21428","title":"Decoupled DiLoCo for Resilient Distributed Pre-training","created":"2026-04-23","cited_by_count":4,"area":null},{"id":"2505.21684","title":"Incentivizing Permissionless Distributed Learning of LLMs","created":"2025-05-27","cited_by_count":3,"area":null},{"id":"2507.07765","title":"Distributed and Decentralised Training: Technical Governance Challenges in a Shifting AI Landscape","created":"2025-07-10","cited_by_count":3,"area":null},{"id":"2507.09029","title":"Model Parallelism With Subnetwork Data Parallelism","created":"2025-07-11","cited_by_count":3,"area":null},{"id":"2510.03434","title":"Paris: A Decentralized Trained Open-Weight Diffusion Model","created":"2025-10-03","cited_by_count":3,"area":"diffusion"},{"id":"2511.20629","title":"MapReduce LoRA: Advancing the Pareto Front in Multi-Preference Optimization for Generative Models","created":"2025-11-25","cited_by_count":3,"area":"alignment"},{"id":"2601.22442","title":"AsyncMesh: Fully Asynchronous Optimization for Data and Pipeline Parallelism","created":"2026-01-30","cited_by_count":3,"area":null},{"id":"2605.11172","title":"Optimistic Dual Averaging Unifies Modern Optimizers","created":"2026-05-11","cited_by_count":3,"area":null},{"id":"2312.02204","title":"Meta-learning Optimizers for Communication-Efficient Learning","created":"2023-12-02","cited_by_count":2,"area":null},{"id":"2404.15834","title":"FEDSTR: Money-In AI-Out | A Decentralized Marketplace for Federated Learning and LLM Training on the NOSTR Protocol","created":"2024-04-15","cited_by_count":2,"area":null},{"id":"2409.13198","title":"Exploring Scaling Laws for Local SGD in Large Language Model Training","created":"2024-09-20","cited_by_count":2,"area":null},{"id":"2507.06931","title":"DICE: Data Influence Cascade in Decentralized Learning","created":"2025-07-09","cited_by_count":2,"area":null},{"id":"2507.10618","title":"Compute Requirements for Algorithmic Innovation in Frontier AI Models","created":"2025-07-13","cited_by_count":2,"area":null},{"id":"2510.00236","title":"Per-example gradients: a new frontier for understanding and improving optimizers","created":"2025-09-30","cited_by_count":2,"area":null},{"id":"2510.05361","title":"MT-DAO: Multi-Timescale Distributed Adaptive Optimizers with Local Updates","created":"2025-10-06","cited_by_count":2,"area":null},{"id":"2510.19470","title":"HybridEP: Scaling Expert Parallelism to Cross-Datacenter Scenario via Hybrid Expert/Data Transmission","created":"2025-10-22","cited_by_count":2,"area":"moe"},{"id":"2602.01746","title":"Rethinking LoRA for Data Heterogeneous Federated Learning: Subspace and State Alignment","created":"2026-02-02","cited_by_count":2,"area":null},{"id":"2602.08019","title":"The Rise of Sparse Mixture-of-Experts: A Survey from Algorithmic Foundations to Decentralized Architectures and Vertical Domain Applications","created":"2026-02-08","cited_by_count":2,"area":"moe"},{"id":"2606.22768","title":"Factored Gossip DiLoCo: Reducing Blocking Communication in DiLoCo","created":"2026-06-22","cited_by_count":2,"area":null},{"id":"2412.07971","title":"Effectiveness of Distributed Gradient Descent with Local Steps for Overparameterized Models","created":"2024-12-10","cited_by_count":1,"area":null},{"id":"2502.13811","title":"On the Duality between Gradient Transformations and Adapters","created":"2025-02-19","cited_by_count":1,"area":null},{"id":"2506.02965","title":"PC-MoE: Memory-Efficient and Privacy-Preserving Collaborative Training for Mixture-of-Experts LLMs","created":"2025-06-03","cited_by_count":1,"area":"moe"},{"id":"2507.17346","title":"Taming Latency and Bandwidth: A Theoretical Framework and Adaptive Algorithm for Communication-Constrained Training","created":"2025-07-23","cited_by_count":1,"area":null},{"id":"2509.19396","title":"OmniFed: A Modular Framework for Configurable Federated Learning from Edge to HPC","created":"2025-09-23","cited_by_count":1,"area":null},{"id":"2509.22418","title":"Partial Parameter Updates for Efficient Distributed Training","created":"2025-09-26","cited_by_count":1,"area":null},{"id":"2511.10683","title":"LT-Soups: Bridging Head and Tail Classes via Subsampled Model Soups","created":"2025-11-11","cited_by_count":1,"area":null},{"id":"2512.13728","title":"CurvaDion: Curvature-Adaptive Distributed Orthonormalization","created":"2025-12-13","cited_by_count":1,"area":null},{"id":"2601.02360","title":"Heterogeneous Low-Bandwidth Pre-Training of LLMs","created":"2026-01-05","cited_by_count":1,"area":null},{"id":"2601.22124","title":"Toward Federated Large Language Models in Medicine: A Parameter-Efficient Framework for Privacy-Preserving, Multi-Institutional Adaptation","created":"2026-01-29","cited_by_count":1,"area":null},{"id":"2602.04396","title":"LoRDO: Distributed Low-Rank Optimization with Infrequent Communication","created":"2026-02-04","cited_by_count":1,"area":null},{"id":"2603.26954","title":"High dimensional theory of two-phase optimizers","created":"2026-03-27","cited_by_count":1,"area":null},{"id":"2605.20866","title":"LOSCAR-SGD: Local SGD with Communication-Computation Overlap and Delay-Corrected Sparse Model Averaging","created":"2026-05-20","cited_by_count":1,"area":null},{"id":"2606.05484","title":"Learned Subspace Compression for Communication-Efficient Pipeline Parallelism","created":"2026-06-03","cited_by_count":1,"area":null},{"id":"2606.11081","title":"Unifying Local Communications and Local Updates for LLM Pretraining","created":"2026-06-09","cited_by_count":1,"area":null},{"id":"2403.16125","title":"Arena: Efficiently Training Large Models via Dynamic Scheduling and Adaptive Parallelism Co-Design","created":"2024-03-24","cited_by_count":0,"area":null},{"id":"2411.02908","title":"Photon: Federated LLM Pre-Training","created":"2024-11-05","cited_by_count":0,"area":null},{"id":"2411.16303","title":"Understanding Generalization of Federated Learning: the Trade-off between Model Stability and Optimization","created":"2024-11-25","cited_by_count":0,"area":null},{"id":"2412.07210","title":"EDiT: A Local-SGD-Based Efficient Distributed Training Method for Large Language Models","created":"2024-12-10","cited_by_count":0,"area":null},{"id":"2502.06728","title":"DeToNATION: Decoupled Torch Network-Aware Training on Interlinked Online Nodes","created":"2025-02-10","cited_by_count":0,"area":null},{"id":"2504.18454","title":"Pseudo-Asynchronous Local SGD: Robust and Efficient Data-Parallel Training","created":"2025-04-25","cited_by_count":0,"area":null},{"id":"2509.12849","title":"AI Factories: It's time to rethink the Cloud-HPC divide","created":"2025-09-16","cited_by_count":0,"area":null},{"id":"2511.20168","title":"On the Limits of Momentum in Decentralized and Federated Optimization","created":"2025-11-25","cited_by_count":0,"area":null},{"id":"2602.11543","title":"Pretraining A Large Language Model using Distributed GPUs: A Memory-Efficient Decentralized Paradigm","created":"2026-02-12","cited_by_count":0,"area":"moe"},{"id":"2603.12596","title":"Optimize Wider, Not Deeper: Consensus Aggregation for Policy Optimization","created":"2026-03-13","cited_by_count":0,"area":"deeprl"},{"id":"2603.25813","title":"MAGNET: Autonomous Expert Model Generation via Decentralized Autoresearch and BitNet Training","created":"2026-03-26","cited_by_count":0,"area":null},{"id":"2604.21072","title":"Distributed Generative Inference of LLM at Internet Scales with Multi-Dimensional Communication Optimization","created":"2026-04-22","cited_by_count":0,"area":"inference"},{"id":"2604.24708","title":"Scalable Hyperparameter-Divergent Ensemble Training with Automatic Learning Rate Exploration for Large Models","created":"2026-04-27","cited_by_count":0,"area":null},{"id":"2605.09126","title":"Cosine-Gated Adam-Decay: Drop-In Staleness-Aware Outer Optimization for Decoupled DiLoCo","created":"2026-05-09","cited_by_count":0,"area":null},{"id":"2605.24326","title":"ScaleAcross Explorer: Exploring Communication Optimization for Scale-Across AI Model Training","created":"2026-05-23","cited_by_count":0,"area":null},{"id":"2605.28585","title":"Outer-Momentum Restarting in High-Dimensional Two-Phase Optimization","created":"2026-05-27","cited_by_count":0,"area":null},{"id":"2605.29359","title":"Does Distributed Training Undermine Compute Governance?","created":"2026-05-28","cited_by_count":0,"area":null},{"id":"2606.01128","title":"Local MixVR: Breaking the Communication-Sample Dependence in Distributed Learning","created":"2026-05-31","cited_by_count":0,"area":null},{"id":"2606.02958","title":"Echelon: Auditable Aggregate-Only Language-Model Adaptation Across Privacy Boundaries","created":"2026-06-01","cited_by_count":0,"area":null},{"id":"2606.16384","title":"Mixtures of Subspaces for Bandwidth Efficient Context Parallel Training","created":"2026-06-15","cited_by_count":0,"area":null},{"id":"2606.19025","title":"FoMoE: Breaking the Full-Replica Barrier with a Federation of MoEs","created":"2026-06-17","cited_by_count":0,"area":"moe"},{"id":"2606.24722","title":"Decentralised AI Training and Inference with BlockTrain","created":"2026-06-23","cited_by_count":0,"area":null},{"id":"2607.00173","title":"TallyTrain: Communication-Efficient Federated Distillation","created":"2026-06-30","cited_by_count":0,"area":null}],"next_cursor":"100"}