{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2faa1c2b-2bfa-5c7d-8cfc-740b3c46c909","task_key":"all--test--2faa1c2b-2bfa-5c7d-8cfc-740b3c46c909","task_revision_id":"1","upstream_id":"","short_description":"Which of the following has the Supreme Court held about the death penalty?","config":"all","split":"test","body":"{\"choices\":\"['The death penalty may only be imposed upon citizens.', 'Under some circumstances, the death penalty may not violate the Eighth Amendment.', 'Under some circumstances, the death penalty may violate the Third Amendment.', 'States may execute any adult regardless of intellectual disability.']\",\"question\":\"Which of the following has the Supreme Court held about the death penalty?\",\"subject\":\"high_school_government_and_politics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}