{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"08d5f14e-0521-53bb-8380-c5d86a2dccfb","task_key":"all--test--08d5f14e-0521-53bb-8380-c5d86a2dccfb","task_revision_id":"1","upstream_id":"","short_description":"Sutherland's study of the 'professional thief' suggested that:","config":"all","split":"test","body":"{\"choices\":\"['people are socialized into a life of crime by associating with others who define it in positive terms', 'the majority of crime is committed by middle class people in professional occupations', 'those who were arrested and charged with theft did not define themselves as thieves', 'the most dangerous criminals on the street were those who were highly skilled thieves']\",\"question\":\"Sutherland's study of the 'professional thief' suggested that:\",\"subject\":\"sociology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}