{"total":204,"rows":[{"id":"2403.03507","title":"GaLore: Memory-Efficient LLM Training by Gradient Low-Rank Projection","created":"2024-03-06","cited_by_count":463,"area":null},{"id":"2402.11505","title":"Federated Fine-tuning of Large Language Models under Heterogeneous Tasks and Client Resources","created":"2024-02-18","cited_by_count":93,"area":null},{"id":"2403.17919","title":"LISA: Layerwise Importance Sampling for Memory-Efficient Large Language Model Fine-Tuning","created":"2024-03-26","cited_by_count":89,"area":null},{"id":"2407.11046","title":"A Survey on LoRA of Large Language Models","created":"2024-07-08","cited_by_count":67,"area":null},{"id":"2502.05795","title":"The Curse of Depth in Large Language Models","created":"2025-02-09","cited_by_count":62,"area":null},{"id":"2503.20762","title":"ASGO: Adaptive Structured Gradient Optimization","created":"2025-03-26","cited_by_count":57,"area":null},{"id":"2412.05270","title":"APOLLO: SGD-like Memory, AdamW-level Performance","created":"2024-12-06","cited_by_count":54,"area":null},{"id":"2402.11867","title":"LoRA Training in the NTK Regime has No Spurious Local Minima","created":"2024-02-19","cited_by_count":48,"area":null},{"id":"2307.14023","title":"Are Transformers with One Layer Self-Attention Using Low-Rank Weight Matrices Universal Approximators?","created":"2023-07-26","cited_by_count":47,"area":null},{"id":"2406.02214","title":"SLTrain: a sparse plus low-rank approach for parameter and memory efficient pretraining","created":"2024-06-04","cited_by_count":45,"area":null},{"id":"2405.17357","title":"DoRA: Enhancing Parameter-Efficient Fine-Tuning with Dynamic Rank Distribution","created":"2024-05-27","cited_by_count":43,"area":null},{"id":"2410.01623","title":"Fira: Can We Achieve Full-rank Training of LLMs Under Low-rank Constraint?","created":"2024-10-02","cited_by_count":43,"area":null},{"id":"2309.00754","title":"Efficient RLHF: Reducing the Memory Usage of PPO","created":"2023-09-01","cited_by_count":40,"area":"alignment"},{"id":"2408.12857","title":"Memory-Efficient LLM Training with Online Subspace Descent","created":"2024-08-23","cited_by_count":39,"area":null},{"id":"2407.08296","title":"Q-GaLore: Quantized GaLore with INT4 Projection and Layer-Adaptive Low-Rank Gradients","created":"2024-07-11","cited_by_count":38,"area":null},{"id":"2410.16103","title":"LDAdam: Adaptive Optimization from Low-Dimensional Gradient Statistics","created":"2024-10-21","cited_by_count":38,"area":null},{"id":"2412.13795","title":"Mix-LN: Unleashing the Power of Deeper Layers by Combining Pre-LN and Post-LN","created":"2024-12-18","cited_by_count":37,"area":null},{"id":"2501.00365","title":"Low-Rank Adaptation for Foundation Models: A Comprehensive Review","created":"2024-12-31","cited_by_count":37,"area":null},{"id":"2501.06425","title":"Tensor Product Attention Is All You Need","created":"2025-01-11","cited_by_count":31,"area":null},{"id":"2402.01376","title":"LoTR: Low Tensor Rank Weight Adaptation","created":"2024-02-02","cited_by_count":29,"area":null},{"id":"2406.06248","title":"Compute Better Spent: Replacing Dense Layers with Structured Matrices","created":"2024-06-10","cited_by_count":28,"area":null},{"id":"2501.06842","title":"SPAM: Spike-Aware Adam with Momentum Reset for Stable LLM Training","created":"2025-01-12","cited_by_count":28,"area":null},{"id":"2502.10940","title":"CoLA: Compute-Efficient Pre-Training of LLMs via Low-Rank Activation","created":"2025-02-16","cited_by_count":28,"area":null},{"id":"2404.02827","title":"BAdam: A Memory Efficient Full Parameter Optimization Method for Large Language Models","created":"2024-04-03","cited_by_count":27,"area":null},{"id":"2411.19557","title":"Initialization using Update Approximation is a Silver Bullet for Extremely Efficient Low-Rank Fine-Tuning","created":"2024-11-29","cited_by_count":22,"area":null},{"id":"2409.09510","title":"Comparing Retrieval-Augmentation and Parameter-Efficient Fine-Tuning for Privacy-Preserving Personalization of Large Language Models","created":"2024-09-14","cited_by_count":21,"area":null},{"id":"2412.13148","title":"SWAN: SGD with Normalization and Whitening Enables Stateless LLM Training","created":"2024-12-17","cited_by_count":20,"area":null},{"id":"2506.03133","title":"PoLAR: Polar-Decomposed Low-Rank Adapter Representation","created":"2025-06-03","cited_by_count":20,"area":null},{"id":"2405.17991","title":"VeLoRA: Memory Efficient Training using Rank-1 Sub-Token Projections","created":"2024-05-28","cited_by_count":18,"area":null},{"id":"2406.17660","title":"Grass: Compute Efficient Low-Memory LLM Training with Structured Sparse Gradients","created":"2024-06-25","cited_by_count":18,"area":null},{"id":"2410.17881","title":"AdaRankGrad: Adaptive Gradient-Rank and Moments for Memory-Efficient LLMs Training and Fine-Tuning","created":"2024-10-23","cited_by_count":17,"area":null},{"id":"2406.10973","title":"ExPLoRA: Parameter-Efficient Extended Pre-Training to Adapt Vision Transformers under Domain Shifts","created":"2024-06-16","cited_by_count":16,"area":null},{"id":"2502.01586","title":"SubTrack++ : Gradient Subspace Tracking for Scalable LLM Training","created":"2025-02-03","cited_by_count":16,"area":null},{"id":"2410.15352","title":"CompAct: Compressed Activations for Memory-Efficient LLM Training","created":"2024-10-20","cited_by_count":15,"area":null},{"id":"2410.18965","title":"On the Crucial Role of Initialization for Matrix Factorization","created":"2024-10-24","cited_by_count":15,"area":null},{"id":"2507.19672","title":"Alignment and Safety in Large Language Models: Safety Mechanisms, Training Paradigms, and Emerging Challenges","created":"2025-07-25","cited_by_count":15,"area":"alignment"},{"id":"2409.10927","title":"Propulsion: Steering LLM with Tiny Fine-Tuning","created":"2024-09-17","cited_by_count":14,"area":null},{"id":"2411.05239","title":"ZipNN: Lossless Compression for AI Models","created":"2024-11-07","cited_by_count":14,"area":null},{"id":"2504.20437","title":"GaLore 2: Large-Scale LLM Pre-Training by Gradient Low-Rank Projection","created":"2025-04-29","cited_by_count":14,"area":null},{"id":"2510.01938","title":"StelLA: Subspace Learning in Low-rank Adaptation using Stiefel Manifold","created":"2025-10-02","cited_by_count":14,"area":null},{"id":"2409.19458","title":"Scalable Fine-tuning from Multiple Data Sources: A First-Order Approximation Approach","created":"2024-09-28","cited_by_count":13,"area":null},{"id":"2410.02117","title":"Searching for Efficient Linear Layers over a Continuous Space of Structured Matrices","created":"2024-10-03","cited_by_count":13,"area":"moe"},{"id":"2502.07923","title":"Sign Operator for Coping with Heavy-Tailed Noise in Non-Convex Optimization: High Probability Bounds Under $(L_0, L_1)$-Smoothness","created":"2025-02-11","cited_by_count":13,"area":null},{"id":"2506.14562","title":"AlphaDecay: Module-wise Weight Decay for Heavy-Tailed Balancing in LLMs","created":"2025-06-17","cited_by_count":13,"area":null},{"id":"2405.16528","title":"LoQT: Low-Rank Adapters for Quantized Pretraining","created":"2024-05-26","cited_by_count":12,"area":null},{"id":"2502.12171","title":"GoRA: Gradient-driven Adaptive Low Rank Adaptation","created":"2025-02-13","cited_by_count":12,"area":null},{"id":"2406.10785","title":"ShareLoRA: Parameter Efficient and Robust Large Language Model Fine-tuning via Shared Low-Rank Adaptation","created":"2024-06-16","cited_by_count":11,"area":null},{"id":"2507.02503","title":"Continual Gradient Low-Rank Projection Fine-Tuning for LLMs","created":"2025-07-03","cited_by_count":11,"area":null},{"id":"2410.08598","title":"Parameter-Efficient Fine-Tuning of Large Language Models using Semantic Knowledge Tuning","created":"2024-10-11","cited_by_count":10,"area":null},{"id":"2411.07837","title":"FRUGAL: Memory-Efficient Optimization by Reducing State Overhead for Scalable Training","created":"2024-11-12","cited_by_count":10,"area":null},{"id":"2503.18225","title":"DeLoRA: Decoupling Angles and Strength in Low-rank Adaptation","created":"2025-03-23","cited_by_count":10,"area":null},{"id":"2508.02668","title":"LOST: Low-rank and Sparse Pre-training for Large Language Models","created":"2025-08-04","cited_by_count":10,"area":null},{"id":"2512.18248","title":"On the Convergence Rate of LoRA Gradient Descent","created":"2025-12-20","cited_by_count":10,"area":null},{"id":"2602.04998","title":"Learning Rate Matters: Vanilla LoRA May Suffice for LLM Fine-tuning","created":"2026-02-04","cited_by_count":10,"area":null},{"id":"2406.17296","title":"BlockLLM: Memory-Efficient Adaptation of LLMs by Selecting and Optimizing the Right Coordinate Blocks","created":"2024-06-25","cited_by_count":9,"area":null},{"id":"2502.06742","title":"Gradient Multi-Normalization for Stateless and Scalable LLM Training","created":"2025-02-10","cited_by_count":9,"area":null},{"id":"2503.19859","title":"An Overview of Low-Rank Structures in the Training and Adaptation of Large Models","created":"2025-03-25","cited_by_count":9,"area":null},{"id":"2505.18877","title":"RefLoRA: Refactored Low-Rank Adaptation for Efficient Fine-Tuning of Large Models","created":"2025-05-24","cited_by_count":9,"area":null},{"id":"2506.08001","title":"Reparameterized LLM Training via Orthogonal Equivalence Transformation","created":"2025-06-09","cited_by_count":9,"area":null},{"id":"2405.18380","title":"Outlier-weighed Layerwise Sampling for LLM Fine-tuning","created":"2024-05-28","cited_by_count":8,"area":null},{"id":"2408.17163","title":"The Iterative Optimal Brain Surgeon: Faster Sparse Recovery by Leveraging Second-Order Information","created":"2024-08-30","cited_by_count":8,"area":null},{"id":"2412.00071","title":"COAP: Memory-Efficient Training with Correlation-Aware Gradient Projection","created":"2024-11-26","cited_by_count":8,"area":null},{"id":"2502.05539","title":"SSH: Sparse Spectrum Adaptation via Discrete Hartley Transformation","created":"2025-02-08","cited_by_count":7,"area":null},{"id":"2502.05790","title":"Breaking the Frozen Subspace: Importance Sampling for Low-Rank Optimization in LLM Pretraining","created":"2025-02-09","cited_by_count":7,"area":null},{"id":"2505.12532","title":"Exploring Sparsity for Parameter Efficient Fine Tuning Using Wavelets for Vision","created":"2025-05-18","cited_by_count":7,"area":null},{"id":"2507.06558","title":"The Primacy of Magnitude in Low-Rank Adaptation","created":"2025-07-09","cited_by_count":7,"area":null},{"id":"2509.18993","title":"CR-Net: Scaling Parameter-Efficient Training with Cross-Layer Low-Rank Structure","created":"2025-09-23","cited_by_count":7,"area":null},{"id":"2409.07787","title":"Stable Language Model Pre-training by Reducing Embedding Variability","created":"2024-09-12","cited_by_count":6,"area":null},{"id":"2409.12371","title":"Communication-Efficient Federated Low-Rank Update Algorithm and its Connection to Implicit Regularization","created":"2024-09-19","cited_by_count":6,"area":null},{"id":"2410.23261","title":"$100K or 100 Days: Trade-offs when Pre-Training with Academic Resources","created":"2024-10-30","cited_by_count":6,"area":null},{"id":"2411.02063","title":"Scalable Efficient Training of Large Language Models with Low-dimensional Projected Attention","created":"2024-11-04","cited_by_count":6,"area":null},{"id":"2502.01378","title":"CE-LoRA: Computation-Efficient LoRA Fine-Tuning for Language Models","created":"2025-02-03","cited_by_count":6,"area":null},{"id":"2505.21732","title":"LaX: Boosting Low-Rank Training of Foundation Models via Latent Crossing","created":"2025-05-27","cited_by_count":6,"area":null},{"id":"2510.23123","title":"Beyond Higher Rank: Token-wise Input-Output Projections for Efficient Low-Rank Adaptation","created":"2025-10-27","cited_by_count":6,"area":null},{"id":"2602.12429","title":"Stabilizing Native Low-Rank LLM Pretraining","created":"2026-02-12","cited_by_count":6,"area":null},{"id":"2603.16731","title":"Understanding Quantization of Optimizer States in LLM Pre-training: Dynamics of State Staleness and Effectiveness of State Resets","created":"2026-03-17","cited_by_count":6,"area":null},{"id":"2404.15198","title":"Lossless and Near-Lossless Compression for Foundation Models","created":"2024-04-05","cited_by_count":5,"area":null},{"id":"2411.15385","title":"Gradient dynamics for low-rank fine-tuning beyond kernels","created":"2024-11-23","cited_by_count":5,"area":null},{"id":"2412.13860","title":"Domain-adaptative Continual Learning for Low-resource Tasks: Evaluation on Nepali","created":"2024-12-18","cited_by_count":5,"area":null},{"id":"2505.22922","title":"Scalable Parameter and Memory Efficient Pretraining for LLM: Recent Algorithmic Advances and Benchmarking","created":"2025-05-28","cited_by_count":5,"area":null},{"id":"2505.23176","title":"The Panaceas for Improving Low-Rank Decomposition in Communication-Efficient Federated Learning","created":"2025-05-29","cited_by_count":5,"area":null},{"id":"2505.23737","title":"On the Convergence Analysis of Muon","created":"2025-05-29","cited_by_count":5,"area":null},{"id":"2506.14280","title":"Improving LoRA with Variational Learning","created":"2025-06-17","cited_by_count":5,"area":null},{"id":"2506.17475","title":"A geometric framework for momentum-based optimizers for low-rank training","created":"2025-06-20","cited_by_count":5,"area":null},{"id":"2602.09009","title":"ANCRe: Adaptive Neural Connection Reassignment for Efficient Depth Scaling","created":"2026-02-09","cited_by_count":5,"area":null},{"id":"2603.05500","title":"POET-X: Memory-efficient LLM Training by Scaling Orthogonal Transformation","created":"2026-03-05","cited_by_count":5,"area":null},{"id":"2308.14149","title":"Examining User-Friendly and Open-Sourced Large GPT Models: A Survey on Language, Multimodal, and Scientific GPT Models","created":"2023-08-27","cited_by_count":4,"area":null},{"id":"2405.15481","title":"Sparse Spectral Training and Inference on Euclidean and Hyperbolic Neural Networks","created":"2024-05-24","cited_by_count":4,"area":null},{"id":"2507.07694","title":"SAS: Simulated Attention Score","created":"2025-07-10","cited_by_count":4,"area":null},{"id":"2507.08091","title":"Low-rank Momentum Factorization for Memory Efficient Training","created":"2025-07-10","cited_by_count":4,"area":null},{"id":"2508.06953","title":"BoRA: Towards More Expressive Low-Rank Adaptation with Block Diversity","created":"2025-08-09","cited_by_count":4,"area":null},{"id":"2509.05213","title":"An Efficient Subspace Algorithm for Federated Learning on Heterogeneous Data","created":"2025-09-05","cited_by_count":4,"area":null},{"id":"2509.18942","title":"Data Efficient Adaptation in Large Language Models via Continuous Low-Rank Fine-Tuning","created":"2025-09-23","cited_by_count":4,"area":null},{"id":"2511.00051","title":"Calibrating and Rotating: A Unified Framework for Weight Conditioning in PEFT","created":"2025-10-28","cited_by_count":4,"area":null},{"id":"2405.18572","title":"Low-rank finetuning for LLMs: A fairness perspective","created":"2024-05-28","cited_by_count":3,"area":null},{"id":"2410.07771","title":"Full-Rank No More: Low-Rank Weight Training for Modern Speech Recognition Models","created":"2024-10-10","cited_by_count":3,"area":null},{"id":"2412.07003","title":"Understanding Gradient Descent through the Training Jacobian","created":"2024-12-09","cited_by_count":3,"area":null},{"id":"2412.15891","title":"TelcoLM: collecting data, adapting, and benchmarking language models for the telecommunication domain","created":"2024-12-20","cited_by_count":3,"area":null},{"id":"2502.13604","title":"BeamLoRA: Beam-Constraint Low-Rank Adaptation","created":"2025-02-19","cited_by_count":3,"area":null},{"id":"2505.14238","title":"ABBA-Adapters: Efficient and Expressive Fine-Tuning of Foundation Models","created":"2025-05-20","cited_by_count":3,"area":null}],"next_cursor":"100"}