{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"17198b78-8bd0-5b82-98cf-e5a7e9b59423","task_key":"default--test--17198b78-8bd0-5b82-98cf-e5a7e9b59423","task_revision_id":"1","upstream_id":"","short_description":"What is the first name of the South African politician who served as Minister of…","config":"default","split":"test","body":"{\"problem\":\"What is the first name of the South African politician who served as Minister of Higher Education and Training and Minister of Home Affairs under President Jacob Zuma and was Deputy Minister in the Presidency for Women, Youth, and Persons with Disabilities?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}