{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"780607a0-9572-5596-9d51-44a6bbe53bc0","task_key":"default--test--780607a0-9572-5596-9d51-44a6bbe53bc0","task_revision_id":"1","upstream_id":"","short_description":"In what minute was the first try of the game scored in the rugby match between…","config":"default","split":"test","body":"{\"problem\":\"In what minute was the first try of the game scored in the rugby match between Spain and Romania that was part of the 2022 Rugby Europe Championship on February 27, 2022?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}