{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"30828a1a-f1d9-5488-8e5e-593dab06f497","task_key":"all--test--30828a1a-f1d9-5488-8e5e-593dab06f497","task_revision_id":"1","upstream_id":"","short_description":"Which is the hierarchy of U.S. Law?","config":"all","split":"test","body":"{\"choices\":\"['Executive agreements, U.S. Constitution, treaties and federal statutes, state law', 'U.S constitution, treaties and federal statutes, executive agreements, state law', 'Treaties and federal statutes, US Constitution, executive agreements, state law', 'U.S constitution, executive agreements, treaties and federal statutes, state law']\",\"question\":\"Which is the hierarchy of U.S. Law?\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}