{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f92e904b-43c2-5dee-96b1-172a60aec48d","task_key":"default--test--f92e904b-43c2-5dee-96b1-172a60aec48d","task_revision_id":"1","upstream_id":"","short_description":"What was the year and month when The Atlantic reported that the Centers for…","config":"default","split":"test","body":"{\"problem\":\"What was the year and month when The Atlantic reported that the Centers for Disease Control and Prevention (CDC) were conflating the results of two different types of coronavirus tests that diagnose current coronavirus infections and measure whether someone has ever had the virus?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}