{"list":"other","total":81,"rows":[{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"1912.01703","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","created":"2019-12-03","cited_by_count":20305},{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2009.03300","title":"Measuring Massive Multitask Language Understanding","created":"2020-09-07","cited_by_count":7963},{"id":"1602.05629","title":"Communication-Efficient Learning of Deep Networks from Decentralized Data","created":"2016-02-17","cited_by_count":7669},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"1803.05457","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","created":"2018-03-14","cited_by_count":4886},{"id":"1905.07830","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","created":"2019-05-19","cited_by_count":4100},{"id":"2104.08691","title":"The Power of Scale for Parameter-Efficient Prompt Tuning","created":"2021-04-18","cited_by_count":3795},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2205.01068","title":"OPT: Open Pre-trained Transformer Language Models","created":"2022-05-02","cited_by_count":3512},{"id":"1912.04977","title":"Advances and Open Problems in Federated Learning","created":"2019-12-10","cited_by_count":3046},{"id":"2203.15556","title":"Training Compute-Optimal Large Language Models","created":"2022-03-29","cited_by_count":3019},{"id":"1812.06127","title":"Federated Optimization in Heterogeneous Networks","created":"2018-12-14","cited_by_count":2987},{"id":"1911.11641","title":"PIQA: Reasoning about Physical Commonsense in Natural Language","created":"2019-11-26","cited_by_count":2883},{"id":"1909.11942","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","created":"2019-09-26","cited_by_count":2790},{"id":"2101.00027","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","created":"2020-12-31","cited_by_count":2411},{"id":"1905.10044","title":"BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions","created":"2019-05-24","cited_by_count":2183},{"id":"1809.02789","title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","created":"2018-09-08","cited_by_count":2164},{"id":"2211.05100","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","created":"2022-11-09","cited_by_count":1984},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2010.11934","title":"mT5: A massively multilingual pre-trained text-to-text transformer","created":"2020-10-22","cited_by_count":1818},{"id":"1910.02054","title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","created":"2019-10-04","cited_by_count":1301},{"id":"1912.06680","title":"Dota 2 with Large Scale Deep Reinforcement Learning","created":"2019-12-13","cited_by_count":1103},{"id":"2108.12409","title":"Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation","created":"2021-08-27","cited_by_count":1103},{"id":"2003.00295","title":"Adaptive Federated Optimization","created":"2020-02-29","cited_by_count":1046},{"id":"1803.02999","title":"On First-Order Meta-Learning Algorithms","created":"2018-03-08","cited_by_count":1018},{"id":"1606.06031","title":"The LAMBADA dataset: Word prediction requiring a broad discourse context","created":"2016-06-20","cited_by_count":992},{"id":"1604.06174","title":"Training Deep Nets with Sublinear Memory Cost","created":"2016-04-21","cited_by_count":848},{"id":"1902.01046","title":"Towards Federated Learning at Scale: System Design","created":"2019-02-04","cited_by_count":832},{"id":"2204.06745","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","created":"2022-04-14","cited_by_count":713},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"1805.09767","title":"Local SGD Converges Fast and Communicates Little","created":"2018-05-24","cited_by_count":605},{"id":"2007.14390","title":"Flower: A Friendly Federated Learning Research Framework","created":"2020-07-28","cited_by_count":455},{"id":"1905.10497","title":"Fair Resource Allocation in Federated Learning","created":"2019-05-25","cited_by_count":394},{"id":"2305.17493","title":"The Curse of Recursion: Training on Generated Data Makes Models Forget","created":"2023-05-27","cited_by_count":356},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2402.17764","title":"The Era of 1-bit LLMs: All Large Language Models are in 1.58 Bits","created":"2024-02-27","cited_by_count":308},{"id":"1812.06162","title":"An Empirical Model of Large-Batch Training","created":"2018-12-14","cited_by_count":281},{"id":"1802.05799","title":"Horovod: fast and easy distributed deep learning in TensorFlow","created":"2018-02-15","cited_by_count":279},{"id":"1905.03871","title":"Differentially Private Learning with Adaptive Clipping","created":"2019-05-09","cited_by_count":220},{"id":"1808.07217","title":"Don't Use Large Mini-Batches, Use Local SGD","created":"2018-08-22","cited_by_count":202},{"id":"2305.05644","title":"Towards Building the Federated GPT: Federated Instruction Tuning","created":"2023-05-09","cited_by_count":185},{"id":"1611.04482","title":"Practical Secure Aggregation for Federated Learning on User-Held Data","created":"2016-11-14","cited_by_count":178},{"id":"1811.02084","title":"Mesh-TensorFlow: Deep Learning for Supercomputers","created":"2018-11-05","cited_by_count":169},{"id":"2401.00448","title":"Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws","created":"2023-12-31","cited_by_count":159},{"id":"2309.00363","title":"FederatedScope-LLM: A Comprehensive Package for Fine-tuning Large Language Models in Federated Learning","created":"2023-09-01","cited_by_count":149},{"id":"2311.08105","title":"DiLoCo: Distributed Low-Communication Training of Language Models","created":"2023-11-14","cited_by_count":121},{"id":"2302.04062","title":"Machine Learning for Synthetic Data Generation: A Review","created":"2023-02-08","cited_by_count":120},{"id":"2106.15115","title":"Neural Machine Translation for Low-Resource Languages: A Survey","created":"2021-06-29","cited_by_count":109},{"id":"2007.01162","title":"Tilted Empirical Risk Minimization","created":"2020-07-02","cited_by_count":106},{"id":"2308.06522","title":"SLoRA: Federated Parameter Efficient Fine-Tuning of Language Models","created":"2023-08-12","cited_by_count":93},{"id":"2006.07264","title":"Low-resource Languages: A Review of Past Work and Future Challenges","created":"2020-06-12","cited_by_count":91},{"id":"2208.12268","title":"FedPrompt: Communication-Efficient and Privacy Preserving Prompt Tuning in Federated Learning","created":"2022-08-25","cited_by_count":91},{"id":"2212.06470","title":"Position: Considerations for Differentially Private Learning with Large-Scale Public Pretraining","created":"2022-12-13","cited_by_count":88},{"id":"2206.01288","title":"Decentralized Training of Foundation Models in Heterogeneous Environments","created":"2022-06-02","cited_by_count":87},{"id":"2106.07820","title":"On Large-Cohort Training for Federated Learning","created":"2021-06-15","cited_by_count":74},{"id":"2310.15080","title":"Federated Learning of Large Language Models with Parameter-Efficient Prompt Tuning and Adaptive Optimization","created":"2023-10-23","cited_by_count":65},{"id":"2310.10049","title":"FATE-LLM: A Industrial Grade Federated Learning Framework for Large Language Models","created":"2023-10-16","cited_by_count":63},{"id":"2305.11414","title":"Federated Foundation Models: Privacy-Preserving and Collaborative Learning for Large Models","created":"2023-05-19","cited_by_count":50},{"id":"2312.08361","title":"Distributed Inference and Fine-tuning of Large Language Models Over The Internet","created":"2023-12-13","cited_by_count":42},{"id":"2307.09182","title":"Federated Learning for Computationally-Constrained Heterogeneous Devices: A Survey","created":"2023-07-18","cited_by_count":40},{"id":"2403.12503","title":"Securing Large Language Models: Threats, Vulnerabilities and Responsible Practices","created":"2024-03-19","cited_by_count":37},{"id":"2102.00875","title":"Scaling Federated Learning for Fine-tuning of Large Language Models","created":"2021-02-01","cited_by_count":36},{"id":"2206.02291","title":"Pretrained Models for Multilingual Federated Learning","created":"2022-06-06","cited_by_count":30},{"id":"2305.12132","title":"Can Public Large Language Models Help Private Cross-device Federated Learning?","created":"2023-05-20","cited_by_count":30},{"id":"2307.09619","title":"Towards Federated Foundation Models: Scalable Dataset Pipelines for Group-Structured Learning","created":"2023-07-18","cited_by_count":29},{"id":"2401.09135","title":"Asynchronous Local-SGD Training for Language Modeling","created":"2024-01-17","cited_by_count":26},{"id":"2002.02090","title":"Faster On-Device Training Using New Federated Momentum Algorithm","created":"2020-02-06","cited_by_count":25},{"id":"2307.13896","title":"Low-Parameter Federated Learning with Large Language Models","created":"2023-07-26","cited_by_count":23},{"id":"2112.03497","title":"Dataset Geography: Mapping Language Data to Language Users","created":"2021-12-07","cited_by_count":22},{"id":"2110.08133","title":"Trade-offs of Local SGD at Scale: An Empirical Study","created":"2021-10-15","cited_by_count":20},{"id":"2209.07124","title":"The Cost of Training Machine Learning Models over Distributed Data Sources","created":"2022-09-15","cited_by_count":18},{"id":"2207.08988","title":"Training Large-Vocabulary Neural Language Models by Private Federated Learning for Resource-Constrained Devices","created":"2022-07-18","cited_by_count":16},{"id":"2307.10476","title":"What can we learn from Data Leakage and Unlearning for Law?","created":"2023-07-19","cited_by_count":15},{"id":"2403.10616","title":"DiPaCo: Distributed Path Composition","created":"2024-03-15","cited_by_count":10},{"id":"2305.07845","title":"Understanding and Improving Model Averaging in Federated Learning on Heterogeneous Data","created":"2023-05-13","cited_by_count":7},{"id":"2308.16498","title":"Generalised Winograd Schema and its Contextuality","created":"2023-08-31","cited_by_count":4},{"id":"2311.06697","title":"Trusted Source Alignment in Large Language Models","created":"2023-11-12","cited_by_count":4}],"next_cursor":null}