{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2e35872b-8ac7-520a-a7e6-99010b27f681","task_key":"all--test--2e35872b-8ac7-520a-a7e6-99010b27f681","task_revision_id":"1","upstream_id":"","short_description":"Aristotle claims that:","config":"all","split":"test","body":"{\"choices\":\"['virtue is in our power, and so is vice.', 'virtue is in our power, but vice is not.', 'vice is in our power, but virtue is not.', 'neither virtue nor vice is in our power.']\",\"question\":\"Aristotle claims that:\",\"subject\":\"philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}