{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9a8c5189-1b93-5c3d-9894-f8fab60a4d0b","task_key":"default--test--9a8c5189-1b93-5c3d-9894-f8fab60a4d0b","task_revision_id":"1","upstream_id":"","short_description":"In the Method of Counteracting Shrinkage section of the 1900 report \"Clays of…","config":"default","split":"test","body":"{\"problem\":\"In the Method of Counteracting Shrinkage section of the 1900 report \\\"Clays of New York, Their Properties and Uses,\\\" which specific substance is described as possessing all the advantages of quartz as a diluent of shrinkage but has the advantage over it that it does not affect the fusibility of the clay?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}