{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"44f246b9-faa8-5a58-a19b-0701393faa5c","task_key":"default--test--44f246b9-faa8-5a58-a19b-0701393faa5c","task_revision_id":"1","upstream_id":"","short_description":"Marcia Bartusiak traces the term \"black hole\" to which physicist (first name,…","config":"default","split":"test","body":"{\"problem\":\"Marcia Bartusiak traces the term \\\"black hole\\\" to which physicist (first name, middle initial, and surname), who reportedly compared the phenomenon in the early 1960s to the Black Hole of Calcutta, a notorious prison where people entered but never left alive?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}