{"list":"other","total":85,"rows":[{"id":"2107.03374","title":"Evaluating Large Language Models Trained on Code","created":"2021-07-07","cited_by_count":8320},{"id":"2108.07732","title":"Program Synthesis with Large Language Models","created":"2021-08-16","cited_by_count":3637},{"id":"2310.06770","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","created":"2023-10-10","cited_by_count":3373},{"id":"2307.15043","title":"Universal and Transferable Adversarial Attacks on Aligned Language Models","created":"2023-07-27","cited_by_count":3104},{"id":"2303.18223","title":"A Survey of Large Language Models","created":"2023-03-31","cited_by_count":2576},{"id":"2403.07974","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","created":"2024-03-12","cited_by_count":2004},{"id":"2409.12186","title":"Qwen2.5-Coder Technical Report","created":"2024-09-18","cited_by_count":1203},{"id":"2401.14196","title":"DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence","created":"2024-01-25","cited_by_count":1181},{"id":"1809.08887","title":"Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task","created":"2018-09-24","cited_by_count":1139},{"id":"2105.09938","title":"Measuring Coding Challenge Competence With APPS","created":"2021-05-20","cited_by_count":902},{"id":"2102.04664","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","created":"2021-02-09","cited_by_count":744},{"id":"1709.00103","title":"Seq2SQL: Generating Structured Queries from Natural Language using Reinforcement Learning","created":"2017-08-31","cited_by_count":687},{"id":"1909.09436","title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","created":"2019-09-20","cited_by_count":654},{"id":"2406.15877","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","created":"2024-06-22","cited_by_count":546},{"id":"2303.12570","title":"RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation","created":"2023-03-22","cited_by_count":367},{"id":"2310.03302","title":"MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation","created":"2023-10-05","cited_by_count":366},{"id":"2306.03091","title":"RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems","created":"2023-06-05","cited_by_count":353},{"id":"2303.17568","title":"CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X","created":"2023-03-30","cited_by_count":300},{"id":"2401.03065","title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution","created":"2024-01-05","cited_by_count":300},{"id":"2402.14658","title":"OpenCodeInterpreter: Integrating Code Generation with Execution and Refinement","created":"2024-02-22","cited_by_count":247},{"id":"2105.12655","title":"CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks","created":"2021-05-25","cited_by_count":200},{"id":"2311.01964","title":"Don't Make Your LLM an Evaluation Benchmark Cheater","created":"2023-11-03","cited_by_count":194},{"id":"2308.07124","title":"OctoPack: Instruction Tuning Code Large Language Models","created":"2023-08-14","cited_by_count":189},{"id":"1909.05378","title":"CoSQL: A Conversational Text-to-SQL Challenge Towards Cross-Domain Natural Language Interfaces to Databases","created":"2019-09-11","cited_by_count":185},{"id":"2308.01861","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","created":"2023-08-03","cited_by_count":168},{"id":"2210.14868","title":"Multi-lingual Evaluation of Code Generation Models","created":"2022-10-26","cited_by_count":159},{"id":"2403.03163","title":"Design2Code: Benchmarking Multimodal Code Generation for Automated Front-End Engineering","created":"2024-03-05","cited_by_count":152},{"id":"1906.02285","title":"SParC: Cross-Domain Semantic Parsing in Context","created":"2019-06-05","cited_by_count":144},{"id":"2403.18624","title":"Vulnerability Detection with Code Language Models: How Far Are We?","created":"2024-03-27","cited_by_count":139},{"id":"2208.08227","title":"MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation","created":"2022-08-17","cited_by_count":136},{"id":"2010.12773","title":"Structure-Grounded Pretraining for Text-to-SQL","created":"2020-10-24","cited_by_count":126},{"id":"2501.01257","title":"CodeElo: Benchmarking Competition-level Code Generation of LLMs with Human-comparable Elo Ratings","created":"2025-01-02","cited_by_count":115},{"id":"2106.01065","title":"Towards Robustness of Text-to-SQL Models against Synonym Substitution","created":"2021-06-02","cited_by_count":110},{"id":"2106.11455","title":"KaggleDBQA: Realistic Evaluation of Text-to-SQL Parsers","created":"2021-06-22","cited_by_count":106},{"id":"1808.09588","title":"Mapping Language to Code in Programmatic Context","created":"2018-08-29","cited_by_count":104},{"id":"2109.05157","title":"Exploring Underexplored Limitations of Cross-Domain Text-to-SQL Generalization","created":"2021-09-11","cited_by_count":92},{"id":"2411.07781","title":"RedCode: Risky Code Execution and Generation Benchmark for Code Agents","created":"2024-11-12","cited_by_count":90},{"id":"2105.13239","title":"CoSQA: 20,000+ Web Queries for Code Search and Question Answering","created":"2021-05-27","cited_by_count":75},{"id":"2408.06450","title":"Evaluating Language Models for Efficient Code Generation","created":"2024-08-12","cited_by_count":73},{"id":"2404.00599","title":"EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories","created":"2024-03-31","cited_by_count":72},{"id":"2311.10372","title":"A Survey of Large Language Models for Code: Evolution, Benchmarking, and Future Trends","created":"2023-11-17","cited_by_count":66},{"id":"2311.07989","title":"Unifying the Perspectives of NLP and Software Engineering: A Survey on Language Models for Code","created":"2023-11-14","cited_by_count":62},{"id":"2503.06680","title":"FEA-Bench: A Benchmark for Evaluating Repository-Level Code Generation for Feature Implementation","created":"2025-03-09","cited_by_count":58},{"id":"1910.02216","title":"JuICe: A Large Scale Distantly Supervised Dataset for Open Domain Context-based Code Generation","created":"2019-10-05","cited_by_count":49},{"id":"2005.02161","title":"LambdaNet: Probabilistic Type Inference using Graph Neural Networks","created":"2020-04-29","cited_by_count":44},{"id":"2406.07436","title":"McEval: Massively Multilingual Code Evaluation","created":"2024-06-11","cited_by_count":44},{"id":"2406.06647","title":"How Efficient is LLM-Generated Code? A Rigorous & High-Standard Benchmark","created":"2024-06-10","cited_by_count":43},{"id":"2502.06215","title":"LessLeak-Bench: A First Investigation of Data Leakage in LLMs Across 83 Software Engineering Benchmarks","created":"2025-02-10","cited_by_count":37},{"id":"2402.07844","title":"Mercury: A Code Efficiency Benchmark for Code Large Language Models","created":"2024-02-12","cited_by_count":35},{"id":"2306.04556","title":"StudentEval: A Benchmark of Student-Written Prompts for Large Language Models of Code","created":"2023-06-07","cited_by_count":32},{"id":"2408.13001","title":"CRUXEval-X: A Benchmark for Multilingual Code Reasoning, Understanding and Execution","created":"2024-08-23","cited_by_count":31},{"id":"2109.08365","title":"CodeQA: A Question Answering Dataset for Source Code Comprehension","created":"2021-09-17","cited_by_count":30},{"id":"2503.07701","title":"Automated Benchmark Generation for Repository-Level Coding Tasks","created":"2025-03-10","cited_by_count":29},{"id":"2406.06918","title":"HumanEvo: An Evolution-aware Benchmark for More Realistic Evaluation of Repository-level Code Generation","created":"2024-06-11","cited_by_count":28},{"id":"2407.14044","title":"ECCO: Can We Improve Model-Generated Code Efficiency Without Sacrificing Functional Correctness?","created":"2024-07-19","cited_by_count":27},{"id":"2402.14623","title":"RoboScript: Code Generation for Free-Form Manipulation Tasks across Real and Simulation","created":"2024-02-22","cited_by_count":26},{"id":"2405.03786","title":"TOGLL: Correct and Strong Test Oracle Generation with LLMs","created":"2024-05-06","cited_by_count":24},{"id":"2404.00566","title":"CodeBenchGen: Creating Scalable Execution-based Code Generation Benchmarks","created":"2024-03-31","cited_by_count":23},{"id":"2412.05210","title":"Evaluating and Aligning CodeLLMs on Human Preference","created":"2024-12-06","cited_by_count":22},{"id":"2402.08100","title":"Investigating the Impact of Data Contamination of Large Language Models in Text-to-SQL Translation","created":"2024-02-12","cited_by_count":21},{"id":"2405.07990","title":"Plot2Code: A Comprehensive Benchmark for Evaluating Multi-modal Large Language Models in Code Generation from Scientific Plots","created":"2024-05-13","cited_by_count":21},{"id":"2408.13204","title":"DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation","created":"2024-08-23","cited_by_count":21},{"id":"2306.10763","title":"Guiding Language Models of Code with Global Context using Monitors","created":"2023-06-19","cited_by_count":20},{"id":"2410.01999","title":"CodeMMLU: A Multi-Task Benchmark for Assessing Code Understanding & Reasoning Capabilities of CodeLLMs","created":"2024-10-02","cited_by_count":18},{"id":"2406.07411","title":"VersiCode: Towards Version-controllable Code Generation","created":"2024-06-11","cited_by_count":17},{"id":"2410.09812","title":"Unraveling the Potential of Large Language Models in Code Translation: How Far Are We?","created":"2024-10-13","cited_by_count":17},{"id":"2309.01940","title":"CodeApex: A Bilingual Programming Evaluation Benchmark for Large Language Models","created":"2023-09-05","cited_by_count":15},{"id":"2210.14494","title":"CS1QA: A Dataset for Assisting Code-based Question Answering in an Introductory Programming Course","created":"2022-10-26","cited_by_count":11},{"id":"2411.00932","title":"LLMs: A Game-Changer for Software Engineers?","created":"2024-11-01","cited_by_count":11},{"id":"2405.16746","title":"Ecosystem of Large Language Models for Code","created":"2024-05-27","cited_by_count":10},{"id":"2410.23299","title":"FVEval: Understanding Language Model Capabilities in Formal Verification of Digital Hardware","created":"2024-10-15","cited_by_count":10},{"id":"2501.07849","title":"The Invisible Hand: Unveiling Provider Bias in Large Language Models for Code Generation","created":"2025-01-14","cited_by_count":10},{"id":"2501.05396","title":"FairCoder: Probing LLM Bias in High-Stakes Decision Making via Coding Tasks","created":"2025-01-09","cited_by_count":9},{"id":"2409.12866","title":"SpecEval: Evaluating Code Comprehension in Large Language Models via Program Specifications","created":"2024-09-19","cited_by_count":8},{"id":"2408.02450","title":"An Evaluation of Requirements Modeling for Cyber-Physical Systems via LLMs","created":"2024-08-05","cited_by_count":7},{"id":"2409.04475","title":"Revolutionizing Database Q&A with Large Language Models: Comprehensive Benchmark and Evaluation","created":"2024-09-05","cited_by_count":7},{"id":"2411.11582","title":"Exploring LLMs for Verifying Technical System Specifications Against Requirements","created":"2024-11-18","cited_by_count":7},{"id":"2109.03341","title":"Software Vulnerability Detection via Deep Learning over Disaggregated Code Graph Representation","created":"2021-09-07","cited_by_count":5},{"id":"2411.18019","title":"A Real-World Benchmark for Evaluating Fine-Grained Issue Solving Capabilities of Large Language Models","created":"2024-11-27","cited_by_count":3},{"id":"2502.18726","title":"Deep-Bench: Deep Learning Benchmark Dataset for Code Generation","created":"2025-02-26","cited_by_count":3},{"id":"2410.00773","title":"BabelBench: An Omni Benchmark for Code-Driven Analysis of Multimodal and Multistructured Data","created":"2024-10-01","cited_by_count":2},{"id":"2411.18088","title":"There are More Fish in the Sea: Automated Vulnerability Repair via Binary Templates","created":"2024-11-27","cited_by_count":2},{"id":"1911.02564","title":"The role of formalism in system requirements (full version)","created":"2019-11-06","cited_by_count":1},{"id":"2406.06550","title":"ChiBench: a Benchmark Suite for Testing Electronic Design Automation Tools","created":"2024-05-24","cited_by_count":1},{"id":"2411.09683","title":"Towards a Classification of Open-Source ML Models and Datasets for Software Engineering","created":"2024-11-14","cited_by_count":1}],"next_cursor":null}