{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9a0bbe6f-22a9-52a5-87ee-4451426a3411","task_key":"default--test--9a0bbe6f-22a9-52a5-87ee-4451426a3411","task_revision_id":"1","upstream_id":"","short_description":"What was the name of the maid who committed suicide by setting herself on fire…","config":"default","split":"test","body":"{\"problem\":\"What was the name of the maid who committed suicide by setting herself on fire on May 21, 2007, because the former MLA Ashok Veer Vikram Singh exploited her physically, while his wife, a sitting MLA from Bijawar, used to beat her and keep her without salary?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}