{"list":"other","total":55,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"1602.05629","title":"Communication-Efficient Learning of Deep Networks from Decentralized Data","created":"2016-02-17","cited_by_count":7669},{"id":"1812.06127","title":"Federated Optimization in Heterogeneous Networks","created":"2018-12-14","cited_by_count":2987},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"2003.00295","title":"Adaptive Federated Optimization","created":"2020-02-29","cited_by_count":1046},{"id":"1606.06031","title":"The LAMBADA dataset: Word prediction requiring a broad discourse context","created":"2016-06-20","cited_by_count":992},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"1805.09767","title":"Local SGD Converges Fast and Communicates Little","created":"2018-05-24","cited_by_count":605},{"id":"1610.02527","title":"Federated Optimization: Distributed Machine Learning for On-Device Intelligence","created":"2016-10-08","cited_by_count":564},{"id":"2201.11990","title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","created":"2022-01-28","cited_by_count":559},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"1903.03934","title":"Asynchronous Federated Optimization","created":"2019-03-10","cited_by_count":267},{"id":"2104.07857","title":"ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","created":"2021-04-16","cited_by_count":256},{"id":"1905.12322","title":"A Study of BFLOAT16 for Deep Learning Training","created":"2019-05-29","cited_by_count":232},{"id":"1808.07217","title":"Don't Use Large Mini-Batches, Use Local SGD","created":"2018-08-22","cited_by_count":202},{"id":"2106.06639","title":"Federated Learning with Buffered Asynchronous Aggregation","created":"2021-06-11","cited_by_count":190},{"id":"2008.03606","title":"Mime: Mimicking Centralized Stochastic Algorithms in Federated Learning","created":"2020-08-08","cited_by_count":154},{"id":"1811.10792","title":"Stochastic Gradient Push for Distributed Deep Learning","created":"2018-11-27","cited_by_count":152},{"id":"2002.07839","title":"Is Local SGD Better than Minibatch SGD?","created":"2020-02-18","cited_by_count":151},{"id":"1802.07088","title":"i-RevNet: Deep Invertible Networks","created":"2018-02-20","cited_by_count":137},{"id":"2202.09357","title":"ProxSkip: Yes! Local Gradient Steps Provably Lead to Communication Acceleration! Finally!","created":"2022-02-18","cited_by_count":134},{"id":"1910.00643","title":"SlowMo: Improving Communication-Efficient Distributed SGD with Slow Momentum","created":"2019-10-01","cited_by_count":121},{"id":"2311.08105","title":"DiLoCo: Distributed Low-Communication Training of Language Models","created":"2023-11-14","cited_by_count":121},{"id":"2007.01045","title":"DAPPLE: A Pipelined Data Parallel Approach for Training Large Models","created":"2020-07-02","cited_by_count":101},{"id":"1609.08326","title":"Asynchronous Stochastic Gradient Descent with Delay Compensation","created":"2016-09-27","cited_by_count":99},{"id":"1708.01012","title":"On the convergence properties of a $K$-step averaging stochastic gradient descent algorithm for nonconvex optimization","created":"2017-08-03","cited_by_count":92},{"id":"1412.6651","title":"Deep learning with Elastic Averaging SGD","created":"2014-12-20","cited_by_count":90},{"id":"2206.08307","title":"Sharper Convergence Guarantees for Asynchronous SGD for Distributed and Federated Learning","created":"2022-06-16","cited_by_count":79},{"id":"2206.07638","title":"Asynchronous SGD Beats Minibatch SGD Under Arbitrary Delays","created":"2022-06-15","cited_by_count":63},{"id":"2106.10207","title":"Distributed Deep Learning in Open Collaborations","created":"2021-06-18","cited_by_count":58},{"id":"1910.05124","title":"PipeMare: Asynchronous Pipeline Parallel DNN Training","created":"2019-10-09","cited_by_count":55},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52},{"id":"1806.10188","title":"A Tight Convergence Analysis for Stochastic Gradient Descent with Delayed Updates","created":"2018-06-26","cited_by_count":47},{"id":"2302.04869","title":"Reversible Vision Transformers","created":"2023-02-09","cited_by_count":33},{"id":"1605.09774","title":"Asynchrony begets Momentum, with an Application to Deep Learning","created":"2016-05-31","cited_by_count":30},{"id":"1809.02839","title":"Efficient and Robust Parallel DNN Training through Model Parallelism on Multi-GPU Platform","created":"2018-09-08","cited_by_count":28},{"id":"1811.03619","title":"Pipe-SGD: A Decentralized Pipelined SGD Framework for Distributed Deep Net Training","created":"2018-11-08","cited_by_count":28},{"id":"2004.09910","title":"torchgpipe: On-the-fly Pipeline Parallelism for Training Giant Models","created":"2020-04-21","cited_by_count":26},{"id":"2109.05109","title":"Toward Communication Efficient Adaptive Gradient Method","created":"2021-09-10","cited_by_count":25},{"id":"2110.08133","title":"Trade-offs of Local SGD at Scale: An Empirical Study","created":"2021-10-15","cited_by_count":20},{"id":"2109.04522","title":"Asynchronous Iterations in Optimization: New Sequence Results and Sharper Algorithmic Guarantees","created":"2021-09-09","cited_by_count":17},{"id":"2401.16265","title":"CO2: Efficient Distributed Training with Full Communication-Computation Overlap","created":"2024-01-29","cited_by_count":17},{"id":"1706.03471","title":"YellowFin and the Art of Momentum Tuning","created":"2017-06-12","cited_by_count":16},{"id":"2106.07644","title":"A Continuized View on Nesterov Acceleration for Stochastic Gradient Descent and Randomized Gossip","created":"2021-06-10","cited_by_count":15},{"id":"1906.09108","title":"Fully Decoupled Neural Network Learning Using Delayed Gradients","created":"2019-06-21","cited_by_count":12},{"id":"2003.11666","title":"Pipelined Backpropagation at Scale: Training Large Models without Batches","created":"2020-03-25","cited_by_count":11},{"id":"2202.08550","title":"Delay-adaptive step-sizes for asynchronous learning","created":"2022-02-17","cited_by_count":11},{"id":"2002.09539","title":"Overlap Local-SGD: An Algorithmic Approach to Hide Communication Delays in Distributed SGD","created":"2020-02-21","cited_by_count":9},{"id":"2208.00779","title":"DADAO: Decoupled Accelerated Decentralized Asynchronous Optimization","created":"2022-07-26","cited_by_count":8},{"id":"1906.12043","title":"Faster Distributed Deep Net Training: Computation and Communication Decoupled Stochastic Gradient Descent","created":"2019-06-28","cited_by_count":7},{"id":"2003.10579","title":"Slow and Stale Gradients Can Win the Race","created":"2020-03-23","cited_by_count":3},{"id":"2012.03747","title":"Accumulated Decoupled Learning: Mitigating Gradient Staleness in Inter-Layer Model Parallelization","created":"2020-12-03","cited_by_count":2}],"next_cursor":null}