{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"28a3f7bc-ce38-5fd4-8f2d-91857e7f50da","task_key":"all--test--28a3f7bc-ce38-5fd4-8f2d-91857e7f50da","task_revision_id":"1","upstream_id":"","short_description":"A manufacturer created a new brand of fish tackle which they sold throughout the…","config":"all","split":"test","body":"{\"choices\":\"['unconstitutional, because the tackle is sold in interstate commerce.', 'unconstitutional, because it interferes with the sovereign right of state governments to engage in intrastate commerce.', 'constitutional, because the supremacy clause validates laws enacted by Congress.', 'constitutional, because it is within the power of Congress to raise revenue.']\",\"question\":\"A manufacturer created a new brand of fish tackle which they sold throughout the country. Congress now imposes a tax of 14 cents on each set of tackle sold in the United States. This tax is\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}