{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8e95eeb4-1fb8-55a3-81d7-1bfd2588d095","task_key":"all--test--8e95eeb4-1fb8-55a3-81d7-1bfd2588d095","task_revision_id":"1","upstream_id":"","short_description":"If 200 computers with a retail value of $100000 are domestically produced in…","config":"all","split":"test","body":"{\"choices\":\"['$100000 higher because of the computers.', '200 higher because of the computers.', 'unaffected until 2006 when the computers are sold and the figure for GDP in 2005 is revised.', 'higher by the wholesale value of the computers.']\",\"question\":\"If 200 computers with a retail value of $100000 are domestically produced in 2005 but not sold until 2006 then GDP in 2005 is\",\"subject\":\"high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}