{"total":27,"rows":[{"id":"2202.08005","title":"Should You Mask 15% in Masked Language Modeling?","created":"2022-02-16","cited_by_count":118,"area":null},{"id":"2112.10740","title":"Are Large-scale Datasets Necessary for Self-Supervised Pre-training?","created":"2021-12-20","cited_by_count":117,"area":null},{"id":"2204.03044","title":"Fusing finetuned models for better pretraining","created":"2022-04-06","cited_by_count":110,"area":null},{"id":"2211.11567","title":"Neural networks trained with SGD learn distributions of increasing complexity","created":"2022-11-21","cited_by_count":50,"area":null},{"id":"2304.13060","title":"Injecting structural hints: Using language models to study inductive biases in language learning","created":"2023-04-25","cited_by_count":35,"area":null},{"id":"2211.16412","title":"Procedural Image Programs for Representation Learning","created":"2022-11-29","cited_by_count":28,"area":null},{"id":"2206.10139","title":"Insights into Pre-training via Simpler Synthetic Tasks","created":"2022-06-21","cited_by_count":26,"area":null},{"id":"2211.00107","title":"Where to start? Analyzing the potential value of intermediate models","created":"2022-10-31","cited_by_count":23,"area":null},{"id":"2310.19961","title":"ExPT: Synthetic Pretraining for Few-Shot Experimental Design","created":"2023-10-30","cited_by_count":22,"area":null},{"id":"2209.14389","title":"Downstream Datasets Make Surprisingly Good Pretraining Corpora","created":"2022-09-28","cited_by_count":20,"area":null},{"id":"2205.14912","title":"E2S2: Encoding-Enhanced Sequence-to-Sequence Pretraining for Language Understanding and Generation","created":"2022-05-30","cited_by_count":19,"area":null},{"id":"2307.11694","title":"SynerGPT: In-Context Learning for Personalized Drug Synergy Prediction and Drug Design","created":"2023-06-19","cited_by_count":13,"area":null},{"id":"2502.19402","title":"General Intelligence Requires Reward-based Pretraining","created":"2025-02-26","cited_by_count":12,"area":null},{"id":"2212.10173","title":"On the Role of Parallel Data in Cross-lingual Transfer Learning","created":"2022-12-20","cited_by_count":11,"area":null},{"id":"2204.09148","title":"What Makes Instruction Learning Hard? An Investigation and a New Challenge in a Synthetic Environment","created":"2022-04-19","cited_by_count":9,"area":null},{"id":"2212.09864","title":"Synthetic Pre-Training Tasks for Neural Machine Translation","created":"2022-12-19","cited_by_count":8,"area":null},{"id":"2207.00982","title":"Linguistically inspired roadmap for building biologically reliable protein language models","created":"2022-07-03","cited_by_count":7,"area":null},{"id":"2310.00796","title":"SIP: Injecting a Structural Inductive Bias into a Seq2Seq Model by Simulation","created":"2023-10-01","cited_by_count":7,"area":null},{"id":"2109.15101","title":"Compositional generalization in semantic parsing with pretrained transformers","created":"2021-09-30","cited_by_count":5,"area":null},{"id":"2410.05224","title":"Cookbook: A framework for improving LLM generative abilities via programmatic data generating templates","created":"2024-10-07","cited_by_count":3,"area":null},{"id":"2505.15769","title":"Transfer of Structural Knowledge from Synthetic Languages","created":"2025-05-21","cited_by_count":3,"area":null},{"id":"2203.00648","title":"Measuring the Impact of Individual Domain Factors in Self-Supervised Pre-Training","created":"2022-03-01","cited_by_count":2,"area":null},{"id":"2302.08272","title":"Revisiting Hidden Representations in Transfer Learning for Medical Imaging","created":"2023-02-16","cited_by_count":2,"area":null},{"id":"2310.17271","title":"Understanding the Role of Input Token Characters in Language Models: How Does Information Loss Affect Performance?","created":"2023-10-26","cited_by_count":2,"area":null},{"id":"2311.11103","title":"Responsible AI Considerations in Text Summarization Research: A Review of Current Practices","created":"2023-11-18","cited_by_count":2,"area":null},{"id":"2311.17280","title":"Does VLN Pretraining Work with Nonsensical or Irrelevant Instructions?","created":"2023-11-28","cited_by_count":2,"area":null},{"id":"2209.11035","title":"MonoByte: A Pool of Monolingual Byte-level Language Models","created":"2022-09-22","cited_by_count":1,"area":null}],"next_cursor":null}