{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a186276b-bfb7-5b11-8329-38456d38ca20","task_key":"all--test--a186276b-bfb7-5b11-8329-38456d38ca20","task_revision_id":"1","upstream_id":"","short_description":"According to Article III, Section 1, of the United States Constitution, judges…","config":"all","split":"test","body":"{\"choices\":\"['Congress cannot remove federal judges from office', 'federal judges have political independence to fulfill their duties', 'the president has no influence over the composition of the Supreme Court', 'federal judges lack opportunity for advancement or promotion']\",\"question\":\"According to Article III, Section 1, of the United States Constitution, judges \\\"shall hold their offices during good behavior...\\\" A consequence of this clause is that:\",\"subject\":\"miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}