{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6ffe337c-58c7-575a-925f-2b507cbac31d","task_key":"default--test--6ffe337c-58c7-575a-925f-2b507cbac31d","task_revision_id":"1","upstream_id":"","short_description":"From what country were the referee, touch judges, and television match official…","config":"default","split":"test","body":"{\"problem\":\"From what country were the referee, touch judges, and television match official in the rugby match between Spain and Romania that was part of the 2022 Rugby Europe Championship on 27 February 2022?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}