{"list":"other","total":189,"rows":[{"id":"2303.08774","title":"GPT-4 Technical Report","created":"2023-03-15","cited_by_count":18396},{"id":"2107.03374","title":"Evaluating Large Language Models Trained on Code","created":"2021-07-07","cited_by_count":8320},{"id":"2305.20050","title":"Let's Verify Step by Step","created":"2023-05-31","cited_by_count":3789},{"id":"2108.07732","title":"Program Synthesis with Large Language Models","created":"2021-08-16","cited_by_count":3637},{"id":"2310.06770","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","created":"2023-10-10","cited_by_count":3373},{"id":"2403.07974","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","created":"2024-03-12","cited_by_count":2004},{"id":"2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","created":"2022-06-09","cited_by_count":1998},{"id":"2203.07814","title":"Competition-Level Code Generation with AlphaCode","created":"2022-02-08","cited_by_count":1462},{"id":"2305.01210","title":"Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation","created":"2023-05-02","cited_by_count":1391},{"id":"2305.15334","title":"Gorilla: Large Language Model Connected with Massive APIs","created":"2023-05-24","cited_by_count":1332},{"id":"2308.03688","title":"AgentBench: Evaluating LLMs as Agents","created":"2023-08-07","cited_by_count":1267},{"id":"1809.08887","title":"Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task","created":"2018-09-24","cited_by_count":1139},{"id":"2105.09938","title":"Measuring Coding Challenge Competence With APPS","created":"2021-05-20","cited_by_count":902},{"id":"2203.13474","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","created":"2022-03-25","cited_by_count":885},{"id":"2102.04664","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","created":"2021-02-09","cited_by_count":744},{"id":"2305.03111","title":"Can LLM Already Serve as A Database Interface? A BIg Bench for Large-Scale Database Grounded Text-to-SQLs","created":"2023-05-04","cited_by_count":688},{"id":"1709.00103","title":"Seq2SQL: Generating Structured Queries from Natural Language using Reinforcement Learning","created":"2017-08-31","cited_by_count":687},{"id":"1905.13319","title":"MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms","created":"2019-05-30","cited_by_count":662},{"id":"1909.09436","title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","created":"2019-09-20","cited_by_count":654},{"id":"2211.10435","title":"PAL: Program-aided Language Models","created":"2022-11-18","cited_by_count":652},{"id":"2304.08244","title":"API-Bank: A Comprehensive Benchmark for Tool-Augmented LLMs","created":"2023-04-14","cited_by_count":591},{"id":"2406.15877","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","created":"2024-06-22","cited_by_count":546},{"id":"2211.11501","title":"DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation","created":"2022-11-18","cited_by_count":452},{"id":"2308.10620","title":"Large Language Models for Software Engineering: A Systematic Literature Review","created":"2023-08-21","cited_by_count":399},{"id":"2303.12570","title":"RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation","created":"2023-03-22","cited_by_count":367},{"id":"1608.01413","title":"Solving General Arithmetic Word Problems","created":"2016-08-04","cited_by_count":360},{"id":"2306.03091","title":"RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems","created":"2023-06-05","cited_by_count":353},{"id":"2401.03065","title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution","created":"2024-01-05","cited_by_count":300},{"id":"2309.10691","title":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback","created":"2023-09-19","cited_by_count":297},{"id":"2306.13304","title":"ToolQA: A Dataset for LLM Question Answering with External Tools","created":"2023-06-23","cited_by_count":274},{"id":"1909.03496","title":"Devign: Effective Vulnerability Identification by Learning Comprehensive Program Semantics via Graph Neural Networks","created":"2019-09-08","cited_by_count":272},{"id":"2411.07763","title":"Spider 2.0: Evaluating Language Models on Real-World Enterprise Text-to-SQL Workflows","created":"2024-11-12","cited_by_count":232},{"id":"2502.10517","title":"KernelBench: Can LLMs Write Efficient GPU Kernels?","created":"2025-02-14","cited_by_count":225},{"id":"2305.12524","title":"TheoremQA: A Theorem-driven Question Answering dataset","created":"2023-05-21","cited_by_count":192},{"id":"2308.07124","title":"OctoPack: Instruction Tuning Code Large Language Models","created":"2023-08-14","cited_by_count":189},{"id":"1909.05378","title":"CoSQL: A Conversational Text-to-SQL Challenge Towards Cross-Domain Natural Language Interfaces to Databases","created":"2019-09-11","cited_by_count":185},{"id":"2310.11248","title":"CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion","created":"2023-10-17","cited_by_count":183},{"id":"2310.03533","title":"Large Language Models for Software Engineering: Survey and Open Problems","created":"2023-10-05","cited_by_count":181},{"id":"1906.04908","title":"SPoC: Search-based Pseudocode to Code","created":"2019-06-12","cited_by_count":176},{"id":"2308.01861","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","created":"2023-08-03","cited_by_count":168},{"id":"1805.08949","title":"Learning to Mine Aligned Code and Natural Language Pairs from Stack Overflow","created":"2018-05-23","cited_by_count":165},{"id":"2312.14852","title":"TACO: Topics in Algorithmic COde generation dataset","created":"2023-12-22","cited_by_count":160},{"id":"2210.14868","title":"Multi-lingual Evaluation of Code Generation Models","created":"2022-10-26","cited_by_count":159},{"id":"2407.12883","title":"BRIGHT: A Realistic and Challenging Benchmark for Reasoning-Intensive Retrieval","created":"2024-07-16","cited_by_count":158},{"id":"1906.02285","title":"SParC: Cross-Domain Semantic Parsing in Context","created":"2019-06-05","cited_by_count":144},{"id":"2208.08227","title":"MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation","created":"2022-08-17","cited_by_count":136},{"id":"2306.06624","title":"RestGPT: Connecting Large Language Models with Real-World RESTful APIs","created":"2023-06-11","cited_by_count":135},{"id":"2010.12773","title":"Structure-Grounded Pretraining for Text-to-SQL","created":"2020-10-24","cited_by_count":126},{"id":"2503.02951","title":"KodCode: A Diverse, Challenging, and Verifiable Synthetic Dataset for Coding","created":"2025-03-04","cited_by_count":119},{"id":"2009.05617","title":"Unit Test Case Generation with Transformers and Focal Context","created":"2020-09-11","cited_by_count":118},{"id":"2501.01257","title":"CodeElo: Benchmarking Competition-level Code Generation of LLMs with Human-comparable Elo Ratings","created":"2025-01-02","cited_by_count":115},{"id":"2106.01065","title":"Towards Robustness of Text-to-SQL Models against Synonym Substitution","created":"2021-06-02","cited_by_count":110},{"id":"2106.11455","title":"KaggleDBQA: Realistic Evaluation of Text-to-SQL Parsers","created":"2021-06-22","cited_by_count":106},{"id":"2302.07867","title":"Learning Performance-Improving Code Edits","created":"2023-02-15","cited_by_count":106},{"id":"2502.12115","title":"SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?","created":"2025-02-17","cited_by_count":105},{"id":"1808.09588","title":"Mapping Language to Code in Programmatic Context","created":"2018-08-29","cited_by_count":104},{"id":"1802.08979","title":"NL2Bash: A Corpus and Semantic Parser for Natural Language Interface to the Linux Operating System","created":"2018-02-25","cited_by_count":98},{"id":"2203.09095","title":"Automating Code Review Activities by Large-Scale Pre-training","created":"2022-03-17","cited_by_count":98},{"id":"2109.05157","title":"Exploring Underexplored Limitations of Cross-Domain Text-to-SQL Generalization","created":"2021-09-11","cited_by_count":92},{"id":"1602.03001","title":"A Convolutional Attention Network for Extreme Summarization of Source Code","created":"2016-02-09","cited_by_count":91},{"id":"2107.08760","title":"CVEfixes: Automated Collection of Vulnerabilities and Their Fixes from Open-Source Software","created":"2021-07-19","cited_by_count":91},{"id":"2406.14497","title":"CodeRAG-Bench: Can Retrieval Augment Code Generation?","created":"2024-06-20","cited_by_count":91},{"id":"2503.02240","title":"OmniSQL: Synthesizing High-quality Text-to-SQL Data at Scale","created":"2025-03-04","cited_by_count":91},{"id":"2402.14809","title":"CriticBench: Benchmarking LLMs for Critique-Correct Reasoning","created":"2024-02-22","cited_by_count":88},{"id":"2206.06888","title":"CERT: Continual Pre-Training on Sketches for Library-Oriented Code Generation","created":"2022-06-14","cited_by_count":87},{"id":"2212.09248","title":"Natural Language to Code Generation in Interactive Data Science Notebooks","created":"2022-12-19","cited_by_count":87},{"id":"2402.16694","title":"HumanEval-XL: A Multilingual Code Generation Benchmark for Cross-lingual Natural Language Generalization","created":"2024-02-26","cited_by_count":86},{"id":"2110.06773","title":"Leveraging Automated Unit Tests for Unsupervised Code Translation","created":"2021-10-13","cited_by_count":82},{"id":"1902.01954","title":"A Neural Model for Generating Natural Language Summaries of Program Subroutines","created":"2019-02-05","cited_by_count":77},{"id":"2304.11938","title":"Is ChatGPT the Ultimate Programming Assistant -- How far is it?","created":"2023-04-24","cited_by_count":76},{"id":"2401.04621","title":"DebugBench: Evaluating Debugging Capability of Large Language Models","created":"2024-01-09","cited_by_count":76},{"id":"2105.13239","title":"CoSQA: 20,000+ Web Queries for Code Search and Question Answering","created":"2021-05-27","cited_by_count":75},{"id":"2308.10335","title":"Can ChatGPT replace StackOverflow? A Study on Robustness and Reliability of Large Language Model Code Generation","created":"2023-08-20","cited_by_count":73},{"id":"2404.00599","title":"EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories","created":"2024-03-31","cited_by_count":72},{"id":"2405.19856","title":"DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories","created":"2024-05-30","cited_by_count":72},{"id":"2205.10583","title":"Automated Repair of Programs from Large Language Models","created":"2022-05-21","cited_by_count":71},{"id":"2008.06448","title":"Loghub: A Large Collection of System Log Datasets for AI-driven Log Analytics","created":"2020-08-14","cited_by_count":70},{"id":"2301.09043","title":"CodeScore: Evaluating Code Generation by Learning Code Execution","created":"2023-01-22","cited_by_count":67},{"id":"2311.09835","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","created":"2023-11-16","cited_by_count":65},{"id":"2402.02037","title":"EffiBench: Benchmarking the Efficiency of Automatically Generated Code","created":"2024-02-03","cited_by_count":64},{"id":"2303.09384","title":"LLMSecEval: A Dataset of Natural Language Prompts for Security Evaluations","created":"2023-03-16","cited_by_count":62},{"id":"2407.02883","title":"CoIR: A Comprehensive Benchmark for Code Information Retrieval Models","created":"2024-07-03","cited_by_count":60},{"id":"2108.11590","title":"AVATAR: A Parallel Corpus for Java-Python Program Translation","created":"2021-08-26","cited_by_count":59},{"id":"1707.02275","title":"A parallel corpus of Python functions and documentation strings for automated code documentation and code generation","created":"2017-07-07","cited_by_count":57},{"id":"2310.04951","title":"CodeTransOcean: A Comprehensive Multilingual Benchmark for Code Translation","created":"2023-10-08","cited_by_count":57},{"id":"2403.09032","title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","created":"2024-03-14","cited_by_count":57},{"id":"2206.08474","title":"XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence","created":"2022-06-16","cited_by_count":56},{"id":"2210.17236","title":"When Language Model Meets Private Library","created":"2022-10-31","cited_by_count":54},{"id":"2305.00418","title":"Using Large Language Models to Generate JUnit Tests: An Empirical Study","created":"2023-04-30","cited_by_count":53},{"id":"2305.18607","title":"How Effective Are Neural Networks for Fixing Security Vulnerabilities","created":"2023-05-29","cited_by_count":52},{"id":"2406.11612","title":"Long Code Arena: a Set of Benchmarks for Long-Context Code Models","created":"2024-06-17","cited_by_count":50},{"id":"1910.02216","title":"JuICe: A Large Scale Distantly Supervised Dataset for Open Domain Context-based Code Generation","created":"2019-10-05","cited_by_count":49},{"id":"2102.07995","title":"D2A: A Dataset Built for AI-Based Vulnerability Detection Methods Using Differential Analysis","created":"2021-02-16","cited_by_count":49},{"id":"2309.08221","title":"Exploring the Potential of ChatGPT in Automated Code Refinement: An Empirical Study","created":"2023-09-15","cited_by_count":44},{"id":"2002.05800","title":"On Learning Meaningful Assert Statements for Unit Test Cases","created":"2020-02-13","cited_by_count":42},{"id":"2203.08388","title":"MCoNaLa: A Benchmark for Code Generation from Multiple Natural Languages","created":"2022-03-16","cited_by_count":41},{"id":"2312.00575","title":"Instruction-tuning Aligns LLMs to the Human Brain","created":"2023-12-01","cited_by_count":40},{"id":"2403.19114","title":"Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM","created":"2024-03-28","cited_by_count":40},{"id":"2406.04531","title":"TESTEVAL: Benchmarking Large Language Models for Test Case Generation","created":"2024-06-06","cited_by_count":40},{"id":"2206.01335","title":"Code Generation Tools (Almost) for Free? A Study of Few-Shot, Pre-Trained Language Models on Code","created":"2022-06-02","cited_by_count":39}],"next_cursor":"100"}