{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"594f711f-f425-51cb-8585-7e7fdf7bf83c","task_key":"all--test--594f711f-f425-51cb-8585-7e7fdf7bf83c","task_revision_id":"1","upstream_id":"","short_description":"What do the terms apartheid, ethnic cleansing, and integration all have in…","config":"all","split":"test","body":"{\"choices\":\"['They all occurred in Nazi Germany during World War I.', 'They are all forms of residential segregation.', 'They are all ways to integrate different ethnic populations in neighborhoods.', 'They are different expressions for the same concept.']\",\"question\":\"What do the terms apartheid, ethnic cleansing, and integration all have in common?\",\"subject\":\"high_school_geography\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}