{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0d3e46b9-7caf-5111-bcbf-1ee530717f41","task_key":"default--test--0d3e46b9-7caf-5111-bcbf-1ee530717f41","task_revision_id":"1","upstream_id":"","short_description":"What was the name of the artist who painted the first Duchess and Lady…","config":"default","split":"test","body":"{\"problem\":\"What was the name of the artist who painted the first Duchess and Lady Fitzharding playing cards, which hung in the green drawing room as of 1908, according to \\\"Historic Houses and Their Gardens: Palaces, Castles, Country Places, and Gardens of the Old and New Worlds\\\"?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}