{"list":"other","total":66,"rows":[{"id":"1706.03762","title":"Attention Is All You Need","created":"2017-06-12","cited_by_count":60361},{"id":"1810.04805","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","created":"2018-10-11","cited_by_count":40399},{"id":"1412.6980","title":"Adam: A Method for Stochastic Optimization","created":"2014-12-22","cited_by_count":37240},{"id":"2005.14165","title":"Language Models are Few-Shot Learners","created":"2020-05-28","cited_by_count":32419},{"id":"2010.11929","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","created":"2020-10-22","cited_by_count":30186},{"id":"2407.21783","title":"The Llama 3 Herd of Models","created":"2024-07-31","cited_by_count":14715},{"id":"1910.10683","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","created":"2019-10-23","cited_by_count":14599},{"id":"2302.13971","title":"LLaMA: Open and Efficient Foundation Language Models","created":"2023-02-27","cited_by_count":14132},{"id":"2103.14030","title":"Swin Transformer: Hierarchical Vision Transformer using Shifted Windows","created":"2021-03-25","cited_by_count":12139},{"id":"2005.12872","title":"End-to-End Object Detection with Transformers","created":"2020-05-26","cited_by_count":7414},{"id":"2001.08361","title":"Scaling Laws for Neural Language Models","created":"2020-01-23","cited_by_count":6586},{"id":"2006.11477","title":"wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations","created":"2020-06-20","cited_by_count":3905},{"id":"2205.14135","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","created":"2022-05-27","cited_by_count":3673},{"id":"2307.08691","title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","created":"2023-07-17","cited_by_count":2598},{"id":"1909.08053","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","created":"2019-09-17","cited_by_count":1868},{"id":"2005.08100","title":"Conformer: Convolution-augmented Transformer for Speech Recognition","created":"2020-05-16","cited_by_count":1863},{"id":"2006.16668","title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding","created":"2020-06-30","cited_by_count":1631},{"id":"2102.05095","title":"Is Space-Time Attention All You Need for Video Understanding?","created":"2021-02-09","cited_by_count":1620},{"id":"2103.15691","title":"ViViT: A Video Vision Transformer","created":"2021-03-29","cited_by_count":1620},{"id":"2309.07864","title":"The Rise and Potential of Large Language Model Based Agents: A Survey","created":"2023-09-14","cited_by_count":1439},{"id":"1710.03740","title":"Mixed Precision Training","created":"2017-10-10","cited_by_count":1156},{"id":"2104.11227","title":"Multiscale Vision Transformers","created":"2021-04-22","cited_by_count":897},{"id":"1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","created":"2019-04-01","cited_by_count":674},{"id":"2304.11277","title":"PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel","created":"2023-04-21","cited_by_count":660},{"id":"2104.04473","title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","created":"2021-04-09","cited_by_count":632},{"id":"2401.09670","title":"DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving","created":"2024-01-18","cited_by_count":617},{"id":"2311.18677","title":"Splitwise: Efficient generative LLM inference using phase splitting","created":"2023-11-30","cited_by_count":558},{"id":"2310.01889","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","created":"2023-10-03","cited_by_count":457},{"id":"2101.06840","title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","created":"2021-01-18","cited_by_count":341},{"id":"2205.05198","title":"Reducing Activation Recomputation in Large Transformer Models","created":"2022-05-10","cited_by_count":302},{"id":"2309.14509","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","created":"2023-09-25","cited_by_count":245},{"id":"2307.06304","title":"Patch n' Pack: NaViT, a Vision Transformer for any Aspect Ratio and Resolution","created":"2023-07-12","cited_by_count":229},{"id":"1807.05358","title":"Beyond Data and Model Parallelism for Deep Neural Networks","created":"2018-07-14","cited_by_count":179},{"id":"2006.09503","title":"Memory-Efficient Pipeline-Parallel DNN Training","created":"2020-06-16","cited_by_count":134},{"id":"2401.11181","title":"Inference without Interference: Disaggregate LLM Inference for Mixed Downstream Workloads","created":"2024-01-20","cited_by_count":121},{"id":"1806.03377","title":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","created":"2018-06-08","cited_by_count":117},{"id":"2402.15627","title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs","created":"2024-02-23","cited_by_count":112},{"id":"2411.18211","title":"TimeMarker: A Versatile Video-LLM for Long and Short Video Understanding with Superior Temporal Localization Ability","created":"2024-11-27","cited_by_count":70},{"id":"2405.07719","title":"USP: A Unified Sequence Parallelism Approach for Long Context Generative AI","created":"2024-05-13","cited_by_count":61},{"id":"2311.09431","title":"Striped Attention: Faster Ring Attention for Causal Transformers","created":"2023-11-15","cited_by_count":54},{"id":"2211.13878","title":"Galvatron: Efficient Transformer Training over Multiple GPUs Using Automatic Parallelism","created":"2022-11-25","cited_by_count":53},{"id":"2306.10209","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","created":"2023-06-16","cited_by_count":52},{"id":"2411.05222","title":"Don't Look Twice: Faster Video Transformers with Run-Length Tokenization","created":"2024-11-07","cited_by_count":43},{"id":"2105.05720","title":"Breaking the Computation and Communication Abstraction Barrier in Distributed Machine Learning Workloads","created":"2021-05-12","cited_by_count":40},{"id":"2401.07872","title":"The What, Why, and How of Context Length Extension Techniques in Large Language Models -- A Detailed Survey","created":"2024-01-15","cited_by_count":33},{"id":"2310.03294","title":"DISTFLASHATTN: Distributed Memory-efficient Attention for Long-context LLMs Training","created":"2023-10-05","cited_by_count":29},{"id":"2004.09910","title":"torchgpipe: On-the-fly Pipeline Parallelism for Training Giant Models","created":"2020-04-21","cited_by_count":26},{"id":"2411.02327","title":"PPLLaVA: Varied Video Sequence Understanding With Prompt Guidance","created":"2024-11-04","cited_by_count":20},{"id":"2412.01523","title":"FlexSP: Accelerating Large Language Model Training via Flexible Sequence Parallelism","created":"2024-12-02","cited_by_count":19},{"id":"2008.08272","title":"Compiling ONNX Neural Network Models Using MLIR","created":"2020-08-19","cited_by_count":18},{"id":"2406.18485","title":"LoongTrain: Efficient Training of Long-Sequence LLMs with Head-Context Parallelism","created":"2024-06-26","cited_by_count":17},{"id":"2210.14826","title":"tf.data service: A Case for Disaggregating ML Input Data Processing","created":"2022-10-26","cited_by_count":16},{"id":"2307.02031","title":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","created":"2023-07-05","cited_by_count":14},{"id":"2311.10418","title":"DynaPipe: Optimizing Multi-task Training through Dynamic Pipelines","created":"2023-11-17","cited_by_count":12},{"id":"2407.09105","title":"Enhancing Training Efficiency Using Packing with Flash Attention","created":"2024-07-12","cited_by_count":12},{"id":"2409.12963","title":"Interpolating Video-LLMs: Toward Longer-sequence LMMs in a Training-free Manner","created":"2024-09-19","cited_by_count":11},{"id":"2410.08081","title":"Packing Analysis: Packing Is More Appropriate for Large Models or Datasets in Supervised Fine-tuning","created":"2024-10-10","cited_by_count":11},{"id":"2401.09149","title":"InternEvo: Efficient Long-sequence Large Language Model Training via Hybrid Parallelism and Redundant Sharding","created":"2024-01-17","cited_by_count":8},{"id":"2409.03365","title":"Spindle: Efficient Distributed Training of Multi-Task Large Models via Wavefront Scheduling","created":"2024-09-05","cited_by_count":7},{"id":"2410.21316","title":"Deep Optimizer States: Towards Scalable Training of Transformer Models Using Interleaved Offloading","created":"2024-10-26","cited_by_count":7},{"id":"2411.00284","title":"SimpleFSDP: Simpler Fully Sharded Data Parallel with torch.compile","created":"2024-11-01","cited_by_count":7},{"id":"2206.06356","title":"Modern Distributed Data-Parallel Large-Scale Pre-training Strategies For NLP models","created":"2022-06-13","cited_by_count":5},{"id":"2407.12117","title":"MEMO: Fine-grained Tensor Management For Ultra-long Context LLM Training","created":"2024-07-16","cited_by_count":5},{"id":"2410.13333","title":"Malleus: Straggler-Resilient Hybrid Parallel Training of Large-scale Models via Malleable Data and Model Parallelization","created":"2024-10-17","cited_by_count":4},{"id":"2410.19367","title":"BitPipe: Bidirectional Interleaved Pipeline Parallelism for Accelerating Large Models Training","created":"2024-10-25","cited_by_count":3},{"id":"1812.01343","title":"Online scheduling of jobs with favorite machines","created":"2018-12-04","cited_by_count":1}],"next_cursor":null}