{"items":[{"slug":"autogpt-gpt4-backend","names":["AutoGPT (GPT4 backend)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"AutoGPT (GPT4 backend)","organization":"","score":14.4,"metric":"Score in %","setup":"Level 1 (146 questions).","reported_by":"paper","source_url":"https://arxiv.org/html/2311.12983v1","source_label":"Table 4, Level 1 score","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"gaia","formal_name":"GAIA"}}],"noindex":false},{"slug":"claude-4-5-opus-20251101-medium","names":["Claude 4.5 Opus (20251101) (medium)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Claude 4.5 Opus (20251101) (medium)","organization":"Anthropic","score":74.4,"metric":"resolved (%)","setup":"Verified; mini-SWE-agent; Model: claude-opus-4-5-20251101; Org: Anthropic; System: Attempts - 1; Mini: 1.16.0","reported_by":"leaderboard","source_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json","source_label":"Verified leaderboard; 20251124_mini-v1.16.0_claude-opus-4-5-20251101","reported_at":"2025-11-24","checked_at":"2026-09-16","benchmark":{"slug":"swe-bench-verified","formal_name":"SWE-bench Verified"}}],"noindex":false},{"slug":"claude-3-opus","names":["Claude-3-Opus"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Claude-3-Opus","organization":"","score":2.42,"metric":"Success Rate (%)","setup":"Screenshot; overall Ubuntu tasks; max 15 steps; temperature 1.0; top-p 0.9.","reported_by":"paper","source_url":"https://arxiv.org/html/2404.07972v1","source_label":"Table 5, Overall column; Section 4.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"osworld","formal_name":"OSWorld"}}],"noindex":false},{"slug":"claude-opus-4-5-20251101-fc","names":["Claude-Opus-4-5-20251101 (FC)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Claude-Opus-4-5-20251101 (FC)","organization":"Anthropic","score":77.47,"metric":"Overall Acc (%)","setup":"BFCL V4; overall; FC.","reported_by":"leaderboard","source_url":"https://gorilla.cs.berkeley.edu/data_overall.csv","source_label":"Official BFCL V4 data, Overall Acc column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"bfcl","formal_name":"Berkeley Function Calling Leaderboard"}}],"noindex":false},{"slug":"claude-sonnet-4-5-20250929-fc","names":["Claude-Sonnet-4-5-20250929 (FC)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Claude-Sonnet-4-5-20250929 (FC)","organization":"Anthropic","score":73.24,"metric":"Overall Acc (%)","setup":"BFCL V4; overall; FC.","reported_by":"leaderboard","source_url":"https://gorilla.cs.berkeley.edu/data_overall.csv","source_label":"Official BFCL V4 data, Overall Acc column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"bfcl","formal_name":"Berkeley Function Calling Leaderboard"}}],"noindex":false},{"slug":"deepseek-v3","names":["DeepSeek V3","DeepSeek-V3"],"report_count":15,"benchmark_count":14,"reports":[{"model":"DeepSeek-V3","organization":"DeepSeek","score":65.2,"metric":"Pass@1","setup":"Base model; 0-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, HumanEval (Pass@1), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"humaneval","formal_name":"HumanEval"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":75.4,"metric":"Pass@1","setup":"Base model; 3-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, MBPP (Pass@1), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"mbpp","formal_name":"MBPP"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":69.8,"metric":"Acc.","setup":"Base model; 2-shot; output prediction.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, CRUXEval-O (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"cruxeval","formal_name":"CRUXEval"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":67.3,"metric":"Acc.","setup":"Base model; 2-shot; input prediction.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, CRUXEval-I (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"cruxeval","formal_name":"CRUXEval"}},{"model":"DeepSeek V3","organization":"DeepSeek","score":39.2,"metric":"Pass@1","setup":"Chat model; 8K output limit; multiple runs with varying temperatures for benchmarks under 1000 samples.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Chat Model, Standard Benchmarks, AIME 2024 (Pass@1), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"aime-2024","formal_name":"AIME 2024"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":89.3,"metric":"EM","setup":"Base model; 8-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, GSM8K (EM), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"gsm8k","formal_name":"GSM8K"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":61.6,"metric":"EM","setup":"Base model; 4-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, MATH (EM), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"math","formal_name":"MATH"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":79.8,"metric":"EM","setup":"Base model; 8-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, MGSM (EM), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"mgsm","formal_name":"MGSM"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":87.5,"metric":"EM","setup":"Base model; 3-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, BBH (EM), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"bbh","formal_name":"BIG-Bench Hard"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":89,"metric":"F1","setup":"Base model; 3-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, DROP (F1), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"drop","formal_name":"DROP"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":88.9,"metric":"Acc.","setup":"Base model; 10-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, HellaSwag (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"hellaswag","formal_name":"HellaSwag"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":64.4,"metric":"Acc.","setup":"Base model; 5-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, MMLU-Pro (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"mmlu-pro","formal_name":"MMLU-Pro"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":84.9,"metric":"Acc.","setup":"Base model; 5-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, WinoGrande (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"winogrande","formal_name":"WinoGrande"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":87.1,"metric":"Acc.","setup":"Base model; 5-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, MMLU (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"mmlu","formal_name":"MMLU"}},{"model":"DeepSeek-V3","organization":"DeepSeek","score":95.3,"metric":"Acc.","setup":"Base model; 25-shot.","reported_by":"vendor","source_url":"https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V3/main/README.md","source_label":"Base Model, Standard Benchmarks, ARC-Challenge (Acc.), final column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"arc-challenge","formal_name":"ARC-Challenge"}}],"noindex":false},{"slug":"gpt-5-4-high","names":["GPT 5.4 (High)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"GPT 5.4 (High)","organization":"OpenAI","score":0.26,"metric":"Score (%)","setup":"Semi-private set; official leaderboard at release; same system prompt, no tools supplied by the evaluator.","reported_by":"paper","source_url":"https://arxiv.org/html/2603.24621v1","source_label":"Table 2; Section 4.3.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"arc-agi-3","formal_name":"ARC-AGI-3"}}],"noindex":false},{"slug":"gpt-4","names":["GPT-4"],"report_count":1,"benchmark_count":1,"reports":[{"model":"GPT-4","organization":"","score":12.24,"metric":"Success Rate (%)","setup":"A11y tree; overall Ubuntu tasks; max 15 steps; temperature 1.0; top-p 0.9.","reported_by":"paper","source_url":"https://arxiv.org/html/2404.07972v1","source_label":"Table 5, Overall column; Section 4.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"osworld","formal_name":"OSWorld"}}],"noindex":false},{"slug":"gpt-4v","names":["GPT-4V"],"report_count":1,"benchmark_count":1,"reports":[{"model":"GPT-4V","organization":"","score":5.26,"metric":"Success Rate (%)","setup":"Screenshot; overall Ubuntu tasks; max 15 steps; temperature 1.0; top-p 0.9.","reported_by":"paper","source_url":"https://arxiv.org/html/2404.07972v1","source_label":"Table 5, Overall column; Section 4.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"osworld","formal_name":"OSWorld"}}],"noindex":false},{"slug":"gpt-5","names":["GPT-5","gpt-5"],"report_count":2,"benchmark_count":2,"reports":[{"model":"gpt-5","organization":"","score":55.9,"metric":"Accuracy (%)","setup":"BM25 retriever; same tool-use prompt; fixed BrowseComp-Plus corpus.","reported_by":"paper","source_url":"https://arxiv.org/html/2508.06600v1","source_label":"Table 1, BM25 rows","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp-plus","formal_name":"BrowseComp-Plus"}},{"model":"GPT-5","organization":"","score":25.3,"metric":"Accuracy (%)","setup":"Dataset updated April 3, 2025; judge model o3-mini.","reported_by":"leaderboard","source_url":"https://lastexam.ai/","source_label":"Quantitative Results, Accuracy column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"hle","formal_name":"Humanity's Last Exam"}}],"noindex":false},{"slug":"gpt4-plugins","names":["GPT4 + plugins*"],"report_count":1,"benchmark_count":1,"reports":[{"model":"GPT4 + plugins*","organization":"","score":30.3,"metric":"Score in %","setup":"Level 1 (146 questions). Plugins manually selected; source says results cannot be reproduced exactly.","reported_by":"paper","source_url":"https://arxiv.org/html/2311.12983v1","source_label":"Table 4, Level 1 score","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"gaia","formal_name":"GAIA"}}],"noindex":false},{"slug":"gemini-3-pro","names":["Gemini 3 Pro"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Gemini 3 Pro","organization":"","score":38.3,"metric":"Accuracy (%)","setup":"Dataset updated April 3, 2025; judge model o3-mini.","reported_by":"leaderboard","source_url":"https://lastexam.ai/","source_label":"Quantitative Results, Accuracy column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"hle","formal_name":"Humanity's Last Exam"}}],"noindex":false},{"slug":"gemini-3-pro-preview-2025-11-18","names":["Gemini 3 Pro Preview (2025-11-18)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Gemini 3 Pro Preview (2025-11-18)","organization":"Google DeepMind","score":74.2,"metric":"resolved (%)","setup":"Verified; mini-SWE-agent; Model: gemini-3-pro-preview; Org: Google DeepMind; System: Attempts - 1; Mini: 1.15.0","reported_by":"leaderboard","source_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json","source_label":"Verified leaderboard; 20251118_mini-v1.15.0_gemini-3-pro-preview-20251118","reported_at":"2025-11-18","checked_at":"2026-09-16","benchmark":{"slug":"swe-bench-verified","formal_name":"SWE-bench Verified"}}],"noindex":false},{"slug":"gemini-3-1-pro-preview","names":["Gemini 3.1 Pro Preview"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Gemini 3.1 Pro Preview","organization":"Google","score":0.37,"metric":"Score (%)","setup":"Semi-private set; official leaderboard at release; same system prompt, no tools supplied by the evaluator.","reported_by":"paper","source_url":"https://arxiv.org/html/2603.24621v1","source_label":"Table 2; Section 4.3.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"arc-agi-3","formal_name":"ARC-AGI-3"}}],"noindex":false},{"slug":"gemini-3-pro-preview-prompt","names":["Gemini-3-Pro-Preview (Prompt)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Gemini-3-Pro-Preview (Prompt)","organization":"Google","score":72.51,"metric":"Overall Acc (%)","setup":"BFCL V4; overall; Prompt.","reported_by":"leaderboard","source_url":"https://gorilla.cs.berkeley.edu/data_overall.csv","source_label":"Official BFCL V4 data, Overall Acc column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"bfcl","formal_name":"Berkeley Function Calling Leaderboard"}}],"noindex":false},{"slug":"grok-4","names":["Grok 4"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Grok 4","organization":"","score":24.5,"metric":"Accuracy (%)","setup":"Dataset updated April 3, 2025; judge model o3-mini.","reported_by":"leaderboard","source_url":"https://lastexam.ai/","source_label":"Quantitative Results, Accuracy column","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"hle","formal_name":"Humanity's Last Exam"}}],"noindex":false},{"slug":"openhands-gpt-5","names":["OpenHands + GPT-5"],"report_count":1,"benchmark_count":1,"reports":[{"model":"OpenHands + GPT-5","organization":"OpenAI","score":71.8,"metric":"resolved (%)","setup":"Verified; OpenHands; Model: openai/gpt-5-2025-08-07; Org: OpenHands; System: Attempts - 1","reported_by":"leaderboard","source_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json","source_label":"Verified leaderboard; 20250807_openhands_gpt5","reported_at":"2025-08-07","checked_at":"2026-09-16","benchmark":{"slug":"swe-bench-verified","formal_name":"SWE-bench Verified"}}],"noindex":false},{"slug":"opus-4-6-max","names":["Opus 4.6 (Max)"],"report_count":1,"benchmark_count":1,"reports":[{"model":"Opus 4.6 (Max)","organization":"Anthropic","score":0.25,"metric":"Score (%)","setup":"Semi-private set; official leaderboard at release; same system prompt, no tools supplied by the evaluator.","reported_by":"paper","source_url":"https://arxiv.org/html/2603.24621v1","source_label":"Table 2; Section 4.3.1","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"arc-agi-3","formal_name":"ARC-AGI-3"}}],"noindex":false},{"slug":"swe-agent-claude-3-7-sonnet","names":["SWE-agent + Claude 3.7 Sonnet"],"report_count":1,"benchmark_count":1,"reports":[{"model":"SWE-agent + Claude 3.7 Sonnet","organization":"Anthropic","score":48,"metric":"resolved (%)","setup":"Lite; SWE-agent; Model: claude-3-7-sonnet-20250219; Org: SWE-agent; System: Attempts - 1","reported_by":"leaderboard","source_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json","source_label":"Lite leaderboard; 20250226_sweagent_claude-3-7-sonnet-20250219","reported_at":"2025-02-26","checked_at":"2026-09-16","benchmark":{"slug":"swe-bench-lite","formal_name":"SWE-bench Lite"}}],"noindex":false},{"slug":"swe-agent-claude-4-sonnet","names":["SWE-agent + Claude 4 Sonnet"],"report_count":1,"benchmark_count":1,"reports":[{"model":"SWE-agent + Claude 4 Sonnet","organization":"Anthropic","score":56.67,"metric":"resolved (%)","setup":"Lite; SWE-agent; Model: claude-4-sonnet-20250514; Org: SWE-agent; System: Attempts - 1","reported_by":"leaderboard","source_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json","source_label":"Lite leaderboard; 20250526_sweagent_claude-4-sonnet-20250514","reported_at":"2025-05-26","checked_at":"2026-09-16","benchmark":{"slug":"swe-bench-lite","formal_name":"SWE-bench Lite"}}],"noindex":false},{"slug":"gpt-4-1","names":["gpt-4.1"],"report_count":1,"benchmark_count":1,"reports":[{"model":"gpt-4.1","organization":"","score":14.58,"metric":"Accuracy (%)","setup":"BM25 retriever; same tool-use prompt; fixed BrowseComp-Plus corpus.","reported_by":"paper","source_url":"https://arxiv.org/html/2508.06600v1","source_label":"Table 1, BM25 rows","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp-plus","formal_name":"BrowseComp-Plus"}}],"noindex":false},{"slug":"gpt-4o-2024-08-06","names":["gpt-4o-2024-08-06"],"report_count":1,"benchmark_count":1,"reports":[{"model":"gpt-4o-2024-08-06","organization":"OpenAI","score":0.6,"metric":"Accuracy (%)","setup":"Without browsing.","reported_by":"paper","source_url":"https://arxiv.org/html/2504.12516v1","source_label":"Table 3; Section 4.1 model identifiers","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp","formal_name":"BrowseComp"}}],"noindex":false},{"slug":"gpt-4o-search-preview-2025-03-11","names":["gpt-4o-search-preview-2025-03-11"],"report_count":1,"benchmark_count":1,"reports":[{"model":"gpt-4o-search-preview-2025-03-11","organization":"OpenAI","score":1.9,"metric":"Accuracy (%)","setup":"With browsing.","reported_by":"paper","source_url":"https://arxiv.org/html/2504.12516v1","source_label":"Table 3; Section 4.1 model identifiers","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp","formal_name":"BrowseComp"}}],"noindex":false},{"slug":"o1-2024-12-17","names":["o1-2024-12-17"],"report_count":1,"benchmark_count":1,"reports":[{"model":"o1-2024-12-17","organization":"OpenAI","score":9.9,"metric":"Accuracy (%)","setup":"Without browsing; medium reasoning effort.","reported_by":"paper","source_url":"https://arxiv.org/html/2504.12516v1","source_label":"Table 3; Section 4.1 model identifiers","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp","formal_name":"BrowseComp"}}],"noindex":false},{"slug":"o3","names":["o3"],"report_count":1,"benchmark_count":1,"reports":[{"model":"o3","organization":"","score":49.28,"metric":"Accuracy (%)","setup":"BM25 retriever; same tool-use prompt; fixed BrowseComp-Plus corpus.","reported_by":"paper","source_url":"https://arxiv.org/html/2508.06600v1","source_label":"Table 1, BM25 rows","reported_at":"","checked_at":"2026-09-16","benchmark":{"slug":"browsecomp-plus","formal_name":"BrowseComp-Plus"}}],"noindex":false}],"next_cursor":null}