{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6bf9bdf5-d16e-54bf-8ee7-79d02e7f1cca","task_key":"default--test--6bf9bdf5-d16e-54bf-8ee7-79d02e7f1cca","task_revision_id":"1","upstream_id":"","short_description":"Despite the Warden of the Kingston Penitentiary claiming the treatments were…","config":"default","split":"test","body":"{\"problem\":\"Despite the Warden of the Kingston Penitentiary claiming the treatments were humane, according to the 1856 report, how many bed deprivations with concurrent bread and water diets were given that year?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}