{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"43206b47-0869-50b3-8eae-583cd52b37df","task_key":"all--test--43206b47-0869-50b3-8eae-583cd52b37df","task_revision_id":"1","upstream_id":"","short_description":"In 1924 the United States Congress passed a bill that significantly altered…","config":"all","split":"test","body":"{\"choices\":\"['suspending further immigration from Latin America', 'making it illegal for employers to hire or recruit undocumented workers', 'promoting family unification, enabling close relatives of people already resident in the United States to be admitted', 'reducing immigration quotas sharply, leading to a dramatic decline in immigration']\",\"question\":\"In 1924 the United States Congress passed a bill that significantly altered immigration patterns by\",\"subject\":\"miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}