{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"922322aa-1d60-593c-a53e-7fb7a027d26a","task_key":"default--test--922322aa-1d60-593c-a53e-7fb7a027d26a","task_revision_id":"1","upstream_id":"","short_description":"What was the muse's name for the sketch \"Serena in the Boat of Apathy,\"…","config":"default","split":"test","body":"{\"problem\":\"What was the muse's name for the sketch \\\"Serena in the Boat of Apathy,\\\" purchased as part of the Oppé Collection with assistance from the National Lottery through the Heritage Lottery Fund in 1996?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}