{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9c2acc15-360d-5ae4-ad7a-ed7c9ed22ae2","task_key":"default--test--9c2acc15-360d-5ae4-ad7a-ed7c9ed22ae2","task_revision_id":"1","upstream_id":"","short_description":"What are the first name, middle name, and surname of the American politician who…","config":"default","split":"test","body":"{\"problem\":\"What are the first name, middle name, and surname of the American politician who served in the California State Assembly from 1967 to 1976 and who died on February 6, 2007?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}