{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6ed785d0-04c0-574e-ba84-1ba3e981c90d","task_key":"default--test--6ed785d0-04c0-574e-ba84-1ba3e981c90d","task_revision_id":"1","upstream_id":"","short_description":"What was the first and last name of the child who won NBC's (National…","config":"default","split":"test","body":"{\"problem\":\"What was the first and last name of the child who won NBC's (National Broadcasting Company) promotional contest in 1950 to find the child who looked the most like TV star puppet Howdy Doody?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}