{"list":"other","total":58,"rows":[{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"1907.11692","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","created":"2019-07-26","cited_by_count":13372},{"id":"1602.05629","title":"Communication-Efficient Learning of Deep Networks from Decentralized Data","created":"2016-02-17","cited_by_count":7669},{"id":"1804.07461","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","created":"2018-04-20","cited_by_count":4899},{"id":"1606.05250","title":"SQuAD: 100,000+ Questions for Machine Comprehension of Text","created":"2016-06-16","cited_by_count":4148},{"id":"1802.05365","title":"Deep contextualized word representations","created":"2018-02-15","cited_by_count":2290},{"id":"1907.02189","title":"On the Convergence of FedAvg on Non-IID Data","created":"2019-07-04","cited_by_count":1229},{"id":"2003.00295","title":"Adaptive Federated Optimization","created":"2020-02-29","cited_by_count":1046},{"id":"1812.01097","title":"LEAF: A Benchmark for Federated Settings","created":"2018-12-03","cited_by_count":792},{"id":"2302.05442","title":"Scaling Vision Transformers to 22 Billion Parameters","created":"2023-02-10","cited_by_count":694},{"id":"1805.09767","title":"Local SGD Converges Fast and Communicates Little","created":"2018-05-24","cited_by_count":605},{"id":"1705.03122","title":"Convolutional Sequence to Sequence Learning","created":"2017-05-08","cited_by_count":552},{"id":"2107.06917","title":"A Field Guide to Federated Optimization","created":"2021-07-14","cited_by_count":315},{"id":"2402.14905","title":"MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases","created":"2024-02-22","cited_by_count":223},{"id":"1901.06053","title":"A Tail-Index Analysis of Stochastic Gradient Noise in Deep Neural Networks","created":"2019-01-18","cited_by_count":214},{"id":"1708.02182","title":"Regularizing and Optimizing LSTM Language Models","created":"2017-08-07","cited_by_count":205},{"id":"2006.15429","title":"Understanding Gradient Clipping in Private SGD: A Geometric Perspective","created":"2020-06-27","cited_by_count":142},{"id":"2311.08105","title":"DiLoCo: Distributed Low-Communication Training of Language Models","created":"2023-11-14","cited_by_count":121},{"id":"2005.10785","title":"Stochastic Optimization with Heavy-Tailed Noise via Accelerated Gradient Clipping","created":"2020-05-21","cited_by_count":104},{"id":"2310.01082","title":"Linear attention is (maybe) all you need (to understand transformer optimization)","created":"2023-10-02","cited_by_count":98},{"id":"2010.02519","title":"Improved Analysis of Clipping Algorithms for Non-convex Optimization","created":"2020-10-05","cited_by_count":89},{"id":"2305.01588","title":"Revisiting Gradient Clipping: Stochastic bias and tight convergence guarantees","created":"2023-05-02","cited_by_count":83},{"id":"2106.14343","title":"High-probability Bounds for Non-Convex Stochastic Optimization with Heavy Tails","created":"2021-06-28","cited_by_count":80},{"id":"2402.19449","title":"Heavy-Tailed Class Imbalance and Why Adam Outperforms Gradient Descent on Language Models","created":"2024-02-29","cited_by_count":80},{"id":"1611.02189","title":"CoCoA: A General Framework for Communication-Efficient Distributed Optimization","created":"2016-11-07","cited_by_count":69},{"id":"2302.00999","title":"High-Probability Bounds for Stochastic Optimization and Variational Inequalities: the Case of Unbounded Variance","created":"2023-02-02","cited_by_count":61},{"id":"2205.02719","title":"Communication-Efficient Adaptive Federated Learning","created":"2022-05-05","cited_by_count":55},{"id":"2406.09405","title":"Why Warmup the Learning Rate? Underlying Mechanisms and Improvements","created":"2024-06-13","cited_by_count":46},{"id":"1906.09069","title":"First Exit Time Analysis of Stochastic Gradient Descent Under Heavy-Tailed Gradient Noise","created":"2019-06-21","cited_by_count":44},{"id":"2202.05963","title":"Private Adaptive Optimization with Side Information","created":"2022-02-12","cited_by_count":38},{"id":"1910.04209","title":"On the adequacy of untuned warmup for adaptive optimization","created":"2019-10-09","cited_by_count":35},{"id":"2002.05685","title":"Fractional Underdamped Langevin Dynamics: Retargeting SGD with Momentum under Heavy-Tailed Gradient Noise","created":"2020-02-13","cited_by_count":33},{"id":"1907.02707","title":"Algorithms of Robust Stochastic Optimization Based on Mirror Descent Method","created":"2019-07-05","cited_by_count":32},{"id":"2106.02305","title":"Local Adaptivity in Federated Learning: Convergence and Consistency","created":"2021-06-04","cited_by_count":31},{"id":"2210.14355","title":"Parameter-free Regret in High Probability with Heavy Tails","created":"2022-10-25","cited_by_count":28},{"id":"2302.06763","title":"Breaking the Lower Bound with (Little) Structure: Acceleration in Non-Convex Stochastic Optimization with Heavy-Tailed Noise","created":"2023-02-14","cited_by_count":27},{"id":"1911.09030","title":"Local AdaAlter: Communication-Efficient Stochastic Gradient Descent with Adaptive Learning Rates","created":"2019-11-20","cited_by_count":26},{"id":"2401.09135","title":"Asynchronous Local-SGD Training for Language Modeling","created":"2024-01-17","cited_by_count":26},{"id":"2203.09697","title":"Towards Training Billion Parameter Graph Neural Networks for Atomic Simulations","created":"2022-03-18","cited_by_count":25},{"id":"2407.07852","title":"OpenDiLoCo: An Open-Source Framework for Globally Distributed Low-Communication Training","created":"2024-07-10","cited_by_count":24},{"id":"2308.00522","title":"Efficient Federated Learning via Local Adaptive Amended Optimizer with Linear Speedup","created":"2023-07-30","cited_by_count":23},{"id":"2410.23922","title":"Analyzing & Reducing the Need for Learning Rate Warmup in GPT Training","created":"2024-10-31","cited_by_count":22},{"id":"2302.05437","title":"High Probability Convergence of Clipped-SGD Under Heavy-tailed Noise","created":"2023-02-10","cited_by_count":21},{"id":"2311.04161","title":"Breaking the Heavy-Tailed Noise Barrier in Stochastic Optimization Problems","created":"2023-11-07","cited_by_count":21},{"id":"2205.05040","title":"A Communication-Efficient Distributed Gradient Clipping Algorithm for Training Deep Neural Networks","created":"2022-05-10","cited_by_count":20},{"id":"2212.00309","title":"Differentially Private Adaptive Optimization with Delayed Preconditioners","created":"2022-12-01","cited_by_count":18},{"id":"2206.01095","title":"Clipped Stochastic Methods for Variational Inequalities with Heavy-Tailed Noise","created":"2022-06-02","cited_by_count":14},{"id":"2311.14632","title":"Differentially Private SGD Without Clipping Bias: An Error-Feedback Approach","created":"2023-11-24","cited_by_count":14},{"id":"2305.18929","title":"Clip21: Error Feedback for Gradient Clipping","created":"2023-05-30","cited_by_count":11},{"id":"2210.00690","title":"Taming Fat-Tailed (\"Heavier-Tailed'' with Potentially Infinite Variance) Noise in Federated Learning","created":"2022-10-03","cited_by_count":10},{"id":"1810.07766","title":"Distributed Learning over Unreliable Networks","created":"2018-10-17","cited_by_count":9},{"id":"2208.08567","title":"High Probability Bounds for Stochastic Subgradient Schemes with Heavy Tailed Noise","created":"2022-08-17","cited_by_count":9},{"id":"2205.15172","title":"Billions of Parameters Are Worth More Than In-domain Training Data: A case study in the Legal Case Entailment Task","created":"2022-05-30","cited_by_count":8},{"id":"2310.06003","title":"Rethinking Memory and Communication Cost for Efficient Large Language Model Training","created":"2023-10-09","cited_by_count":4},{"id":"2310.16920","title":"Smoothed Gradient Clipping and Error Feedback for Decentralized Optimization under Symmetric Heavy-Tailed Noise","created":"2023-10-25","cited_by_count":4},{"id":"2410.18117","title":"Efficient Adaptive Federated Optimization","created":"2024-10-10","cited_by_count":2},{"id":"2410.20135","title":"Near-Optimal Streaming Heavy-Tailed Statistical Estimation with Clipped SGD","created":"2024-10-26","cited_by_count":1}],"next_cursor":null}