{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"67380f8d-418c-50ee-a85a-b9aa505a115e","task_key":"default--test--67380f8d-418c-50ee-a85a-b9aa505a115e","task_revision_id":"1","upstream_id":"","short_description":"In which month and year was Satyanarayan Gangaram Pitroda (an Indian…","config":"default","split":"test","body":"{\"problem\":\"In which month and year was Satyanarayan Gangaram Pitroda (an Indian telecommunication engineer and entrepreneur) appointed as advisor to Indian Prime Minister Manmohan Singh on Public Information Infrastructure and Innovations with the rank of Cabinet Minister?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}