{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"370a3f16-0dc2-5e62-a37c-dd871d04710b","task_key":"default--test--370a3f16-0dc2-5e62-a37c-dd871d04710b","task_revision_id":"1","upstream_id":"","short_description":"On which date, month, and year did the BBC publish an apology on their website…","config":"default","split":"test","body":"{\"problem\":\"On which date, month, and year did the BBC publish an apology on their website for previously publishing inaccurate statements about Tunisian politician Rached Ghannouchi six months earlier on 21 November 2012?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}