{"total":217,"rows":[{"id":"2403.03507","title":"GaLore: Memory-Efficient LLM Training by Gradient Low-Rank Projection","created":"2024-03-06","cited_by_count":463,"area":null},{"id":"2205.10770","title":"Memorization Without Overfitting: Analyzing the Training Dynamics of Large Language Models","created":"2022-05-22","cited_by_count":220,"area":null},{"id":"2109.02934","title":"Fishr: Invariant Gradient Variances for Out-of-Distribution Generalization","created":"2021-09-07","cited_by_count":186,"area":null},{"id":"2407.05000","title":"LoRA-GA: Low-Rank Adaptation with Gradient Approximation","created":"2024-07-06","cited_by_count":141,"area":null},{"id":"2406.16793","title":"Adam-mini: Use Fewer Learning Rates To Gain More","created":"2024-06-24","cited_by_count":137,"area":null},{"id":"2402.16788","title":"Why Transformers Need Adam: A Hessian Perspective","created":"2024-02-26","cited_by_count":124,"area":null},{"id":"2106.09352","title":"Large Scale Private Learning via Low-rank Reparametrization","created":"2021-06-17","cited_by_count":88,"area":null},{"id":"2102.12677","title":"Do Not Let Privacy Overbill Utility: Gradient Embedding Perturbation for Private Learning","created":"2021-02-25","cited_by_count":86,"area":null},{"id":"2212.06152","title":"Accelerating Dataset Distillation via Model Augmentation","created":"2022-12-12","cited_by_count":62,"area":null},{"id":"2503.20762","title":"ASGO: Adaptive Structured Gradient Optimization","created":"2025-03-26","cited_by_count":57,"area":null},{"id":"2209.14981","title":"Stop Wasting My Time! Saving Days of ImageNet and BERT Training with Latest Weight Averaging","created":"2022-09-29","cited_by_count":54,"area":null},{"id":"2106.16225","title":"Analytic Insights into Structure and Rank of Neural Network Hessian Maps","created":"2021-06-30","cited_by_count":52,"area":null},{"id":"2202.01889","title":"Generalizing to New Physical Systems via Context-Informed Dynamics Model","created":"2022-02-01","cited_by_count":50,"area":null},{"id":"2410.07698","title":"Enhancing Zeroth-order Fine-tuning for Language Models with Low-rank Structures","created":"2024-10-10","cited_by_count":50,"area":null},{"id":"2112.00193","title":"Public Data-Assisted Mirror Descent for Private Model Training","created":"2021-12-01","cited_by_count":49,"area":null},{"id":"2207.00160","title":"When Does Differentially Private Learning Not Suffer in High Dimensions?","created":"2022-07-01","cited_by_count":48,"area":null},{"id":"2106.00042","title":"A study on the plasticity of neural networks","created":"2021-05-31","cited_by_count":44,"area":null},{"id":"2111.12229","title":"Subspace Adversarial Training","created":"2021-11-24","cited_by_count":41,"area":null},{"id":"2207.13129","title":"LGV: Boosting Adversarial Example Transferability from Large Geometric Vicinity","created":"2022-07-26","cited_by_count":39,"area":null},{"id":"2306.04251","title":"Stochastic Collapse: How Gradient Noise Attracts SGD Dynamics Towards Simpler Subnetworks","created":"2023-06-07","cited_by_count":39,"area":null},{"id":"2408.12857","title":"Memory-Efficient LLM Training with Online Subspace Descent","created":"2024-08-23","cited_by_count":39,"area":null},{"id":"2405.16002","title":"Does SGD really happen in tiny subspaces?","created":"2024-05-25","cited_by_count":36,"area":null},{"id":"2107.11228","title":"Taxonomizing local versus global structure in neural network loss landscapes","created":"2021-07-23","cited_by_count":34,"area":null},{"id":"2103.01519","title":"Hessian Eigenspectra of More Realistic Nonlinear Models","created":"2021-03-02","cited_by_count":33,"area":null},{"id":"2306.04815","title":"Catapults in SGD: spikes in the training loss and their impact on generalization through feature learning","created":"2023-06-07","cited_by_count":31,"area":null},{"id":"2105.09557","title":"Power-law escape rate of SGD","created":"2021-05-20","cited_by_count":29,"area":null},{"id":"2104.11044","title":"Analyzing Monotonic Linear Interpolation in Neural Network Loss Landscapes","created":"2021-04-22","cited_by_count":28,"area":null},{"id":"2107.05802","title":"How many degrees of freedom do we need to train deep networks: a loss landscape perspective","created":"2021-07-13","cited_by_count":28,"area":null},{"id":"2302.01172","title":"STEP: Learning N:M Structured Sparsity Masks from Scratch with Precondition","created":"2023-02-02","cited_by_count":26,"area":null},{"id":"2310.09639","title":"DPZero: Private Fine-Tuning of Language Models without Backpropagation","created":"2023-10-14","cited_by_count":26,"area":null},{"id":"2312.00246","title":"Directions of Curvature as an Explanation for Loss of Plasticity","created":"2023-11-30","cited_by_count":26,"area":null},{"id":"2405.21064","title":"Recurrent neural networks: vanishing and exploding gradients are not the end of the story","created":"2024-05-31","cited_by_count":25,"area":null},{"id":"2402.17457","title":"Super Consistency of Neural Network Landscapes and Learning Rate Transfer","created":"2024-02-27","cited_by_count":23,"area":null},{"id":"2206.10451","title":"Winning the Lottery Ahead of Time: Efficient Early Network Pruning","created":"2022-06-21","cited_by_count":21,"area":null},{"id":"2404.13588","title":"Machine Unlearning via Null Space Calibration","created":"2024-04-21","cited_by_count":21,"area":"safety"},{"id":"2105.00303","title":"RATT: Leveraging Unlabeled Data to Guarantee Generalization","created":"2021-05-01","cited_by_count":20,"area":null},{"id":"2409.14396","title":"Flat-LoRA: Low-Rank Adaptation over a Flat Loss Landscape","created":"2024-09-22","cited_by_count":20,"area":null},{"id":"2410.07170","title":"Parameter Efficient Fine-tuning via Explained Variance Adaptation","created":"2024-10-09","cited_by_count":20,"area":null},{"id":"2107.09133","title":"The Limiting Dynamics of SGD: Modified Loss, Phase Space Oscillations, and Anomalous Diffusion","created":"2021-07-19","cited_by_count":19,"area":null},{"id":"2305.17446","title":"Fine-tuning Happens in Tiny Subspaces: Exploring Intrinsic Task-specific Subspaces of Pre-trained Language Models","created":"2023-05-27","cited_by_count":19,"area":null},{"id":"2110.14856","title":"An Operator Theoretic View on Pruning Deep Neural Networks","created":"2021-10-28","cited_by_count":18,"area":null},{"id":"2405.17991","title":"VeLoRA: Memory Efficient Training using Rank-1 Sub-Token Projections","created":"2024-05-28","cited_by_count":18,"area":null},{"id":"2302.03764","title":"Sketchy: Memory-efficient Adaptive Regularization with Frequent Directions","created":"2023-02-07","cited_by_count":17,"area":null},{"id":"2106.00001","title":"Privately Learning Subspaces","created":"2021-05-28","cited_by_count":16,"area":null},{"id":"2106.02624","title":"ViViT: Curvature access through the generalized Gauss-Newton's low-rank structure","created":"2021-06-04","cited_by_count":16,"area":null},{"id":"2107.10370","title":"Analytic Study of Families of Spurious Minima in Two-Layer ReLU Neural Networks: A Tale of Symmetry II","created":"2021-07-21","cited_by_count":16,"area":null},{"id":"2201.13011","title":"On the Power-Law Hessian Spectrums in Deep Learning","created":"2022-01-31","cited_by_count":16,"area":null},{"id":"2202.00280","title":"Recycling Model Updates in Federated Learning: Are Gradient Subspaces Low-Rank?","created":"2022-02-01","cited_by_count":16,"area":null},{"id":"2303.01256","title":"Choosing Public Datasets for Private Machine Learning via Gradient Subspace Distance","created":"2023-03-02","cited_by_count":16,"area":null},{"id":"2502.01586","title":"SubTrack++ : Gradient Subspace Tracking for Scalable LLM Training","created":"2025-02-03","cited_by_count":16,"area":null},{"id":"2506.15872","title":"Hidden Breakthroughs in Language Model Training","created":"2025-06-18","cited_by_count":15,"area":null},{"id":"2110.14163","title":"Does the Data Induce Capacity Control in Deep Learning?","created":"2021-10-27","cited_by_count":14,"area":null},{"id":"2210.13447","title":"Precision Machine Learning","created":"2022-10-24","cited_by_count":14,"area":null},{"id":"2212.08272","title":"Communication-Efficient Federated Learning for Heterogeneous Edge Devices Based on Adaptive Gradient Quantization","created":"2022-12-16","cited_by_count":14,"area":null},{"id":"2409.10927","title":"Propulsion: Steering LLM with Tiny Fine-Tuning","created":"2024-09-17","cited_by_count":14,"area":null},{"id":"2201.01448","title":"Conditional Imitation Learning for Multi-Agent Games","created":"2022-01-05","cited_by_count":13,"area":"offlinerl"},{"id":"2212.02083","title":"On the Overlooked Structure of Stochastic Gradients","created":"2022-12-05","cited_by_count":13,"area":null},{"id":"2410.23169","title":"The Persistence of Neural Collapse Despite Low-Rank Bias","created":"2024-10-30","cited_by_count":13,"area":null},{"id":"2505.02809","title":"Towards Quantifying the Hessian Structure of Neural Networks","created":"2025-05-05","cited_by_count":13,"area":null},{"id":"2602.11948","title":"Insights on Muon from Simple Quadratics","created":"2026-02-12","cited_by_count":13,"area":null},{"id":"2405.16528","title":"LoQT: Low-Rank Adapters for Quantized Pretraining","created":"2024-05-26","cited_by_count":12,"area":null},{"id":"2506.01260","title":"Subspace Networks: Scaling Decentralized Training with Communication-Efficient Model Parallelism","created":"2025-06-02","cited_by_count":12,"area":null},{"id":"2211.10691","title":"Two Facets of SDE Under an Information-Theoretic Lens: Generalization of SGD via Training Trajectories and via Terminal States","created":"2022-11-19","cited_by_count":11,"area":null},{"id":"2302.01952","title":"On a continuous time model of gradient descent dynamics and instability in deep learning","created":"2023-02-03","cited_by_count":11,"area":null},{"id":"2404.06106","title":"Unifying Low Dimensional Spectra in Deep Learning","created":"2024-04-09","cited_by_count":11,"area":null},{"id":"2412.19616","title":"Gradient Weight-normalized Low-rank Projection for Efficient LLM Training","created":"2024-12-27","cited_by_count":11,"area":null},{"id":"2102.10734","title":"Provable Super-Convergence with a Large Cyclical Learning Rate","created":"2021-02-22","cited_by_count":10,"area":null},{"id":"2305.09088","title":"The Hessian perspective into the Nature of Convolutional Neural Networks","created":"2023-05-16","cited_by_count":10,"area":null},{"id":"2402.03994","title":"Efficient Sketches for Training Data Attribution and Studying the Loss Landscape","created":"2024-02-06","cited_by_count":10,"area":null},{"id":"2410.08598","title":"Parameter-Efficient Fine-Tuning of Large Language Models using Semantic Knowledge Tuning","created":"2024-10-11","cited_by_count":10,"area":null},{"id":"2602.00170","title":"The Blessing of Dimensionality in LLM Fine-tuning: A Variance-Curvature Perspective","created":"2026-01-30","cited_by_count":10,"area":null},{"id":"2203.11355","title":"Origami in N dimensions: How feed-forward networks manufacture linear separability","created":"2022-03-21","cited_by_count":9,"area":null},{"id":"2302.09160","title":"Identifying Equivalent Training Dynamics","created":"2023-02-17","cited_by_count":9,"area":null},{"id":"2406.03519","title":"Noise-Aware Algorithm for Heterogeneous Differentially Private Federated Learning","created":"2024-06-05","cited_by_count":9,"area":null},{"id":"2501.19183","title":"Position: Curvature Matrices Should Be Democratized via Linear Operators","created":"2025-01-31","cited_by_count":9,"area":null},{"id":"2110.03210","title":"Universality of Winning Tickets: A Renormalization Group Perspective","created":"2021-10-07","cited_by_count":8,"area":null},{"id":"2206.03483","title":"Few-Shot Learning by Dimensionality Reduction in Gradient Space","created":"2022-06-07","cited_by_count":8,"area":null},{"id":"2305.01134","title":"PGrad: Learning Principal Gradients For Domain Generalization","created":"2023-05-02","cited_by_count":8,"area":null},{"id":"2501.12243","title":"FOCUS: First Order Concentrated Updating Scheme","created":"2025-01-21","cited_by_count":8,"area":null},{"id":"2102.03793","title":"Tilting the playing field: Dynamical loss functions for machine learning","created":"2021-02-07","cited_by_count":7,"area":null},{"id":"2102.06604","title":"Cockpit: A Practical Debugging Tool for the Training of Deep Neural Networks","created":"2021-02-12","cited_by_count":7,"area":null},{"id":"2210.17011","title":"A picture of the space of typical learnable tasks","created":"2022-10-31","cited_by_count":7,"area":null},{"id":"2404.03784","title":"A Layer Selection Approach to Test Time Adaptation","created":"2024-04-04","cited_by_count":7,"area":null},{"id":"2405.17088","title":"Phase Transitions in the Output Distribution of Large Language Models","created":"2024-05-27","cited_by_count":7,"area":null},{"id":"2410.16710","title":"Influential Language Data Selection via Gradient Trajectory Pursuit","created":"2024-10-22","cited_by_count":7,"area":null},{"id":"2502.05790","title":"Breaking the Frozen Subspace: Importance Sampling for Low-Rank Optimization in LLM Pretraining","created":"2025-02-09","cited_by_count":7,"area":null},{"id":"2512.22768","title":"Understanding the Mechanisms of Fast Hyperparameter Transfer","created":"2025-12-28","cited_by_count":7,"area":null},{"id":"2103.06733","title":"Intraclass clustering: an implicit learning ability that regularizes DNNs","created":"2021-03-11","cited_by_count":6,"area":null},{"id":"2201.01954","title":"Federated Optimization of Smooth Loss Functions","created":"2022-01-06","cited_by_count":6,"area":null},{"id":"2310.17163","title":"Low-Dimensional Gradient Helps Out-of-Distribution Detection","created":"2023-10-26","cited_by_count":6,"area":null},{"id":"2405.15756","title":"Wasserstein Distances, Neuronal Entanglement, and Sparsity","created":"2024-05-24","cited_by_count":6,"area":"mechinterp"},{"id":"2409.12371","title":"Communication-Efficient Federated Low-Rank Update Algorithm and its Connection to Implicit Regularization","created":"2024-09-19","cited_by_count":6,"area":null},{"id":"2601.23261","title":"TEON: Tensorized Orthonormalization Beyond Layer-Wise Muon for Large Language Model Pre-Training","created":"2026-01-30","cited_by_count":6,"area":null},{"id":"2602.07078","title":"The Optimal Token Baseline: Variance Reduction for Long-Horizon LLM-RL","created":"2026-02-06","cited_by_count":6,"area":null},{"id":"2603.15678","title":"Spectral Edge Dynamics of Training Trajectories: Signal--Noise Geometry Across Scales","created":"2026-03-14","cited_by_count":6,"area":null},{"id":"2605.18632","title":"Position: Weight Space Should Be a First-Class Generative AI Modality","created":"2026-05-18","cited_by_count":6,"area":null},{"id":"2606.04662","title":"Why Muon Outperforms Adam: A Curvature Perspective","created":"2026-06-03","cited_by_count":6,"area":null},{"id":"2103.11154","title":"Low Dimensional Landscape Hypothesis is True: DNNs can be Trained in Tiny Subspaces","created":"2021-03-20","cited_by_count":5,"area":null},{"id":"2303.13635","title":"Low Rank Optimization for Efficient Deep Learning: Making A Balance between Compact Architecture and Fast Training","created":"2023-03-22","cited_by_count":5,"area":null},{"id":"2305.17409","title":"On the special role of class-selective neurons in early training","created":"2023-05-27","cited_by_count":5,"area":null}],"next_cursor":"100"}