{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ecc3e85a-8982-5644-9eb1-eb1578a5b49a","task_key":"default--test--ecc3e85a-8982-5644-9eb1-eb1578a5b49a","task_revision_id":"1","upstream_id":"","short_description":"The drawing of William Buell Sr.'s (1751-1832) house he built, displayed on the…","config":"default","split":"test","body":"{\"problem\":\"The drawing of William Buell Sr.'s (1751-1832) house he built, displayed on the historic plaque installed in 2006 at the intersection of Water Street West and Home Street in Brockville, Ontario, was drawn in 1887 by which Brockville artist?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}