{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"eb21bc84-4e0b-5f8f-bba3-5a7de806df97","task_key":"default--test--eb21bc84-4e0b-5f8f-bba3-5a7de806df97","task_revision_id":"1","upstream_id":"","short_description":"When NASA's communication director reported in 2020 that NASA planned to land…","config":"default","split":"test","body":"{\"problem\":\"When NASA's communication director reported in 2020 that NASA planned to land astronauts on the Moon as part of the U.S. Artemis program, what was the total number of female candidates in the program?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}