{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b41f9e7e-90c8-53b0-a58a-d484d573a812","task_key":"default--test--b41f9e7e-90c8-53b0-a58a-d484d573a812","task_revision_id":"1","upstream_id":"","short_description":"According to \"A Survey of London; Reprinted From the Text of 1603,\" in 1206,…","config":"default","split":"test","body":"{\"problem\":\"According to \\\"A Survey of London; Reprinted From the Text of 1603,\\\" in 1206, 1220, 1224, and 1243, Crown pleas were heard in the Tower of London, with William of York, Richard Passelew, Henry Bathe, and which other justice presiding?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}