{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6cc23bb8-29b4-53f6-8caa-90ad121894f4","task_key":"all--test--6cc23bb8-29b4-53f6-8caa-90ad121894f4","task_revision_id":"1","upstream_id":"","short_description":"A defendant gave a man a gun and instructed him to kill a victim. The man shot…","config":"all","split":"test","body":"{\"choices\":\"['Insanity.', 'Self-defense.', 'Diminished capacity.', 'The Wharton Rule.']\",\"question\":\"A defendant gave a man a gun and instructed him to kill a victim. The man shot and killed the victim with the gun. The man was charged with murder and acquitted. The defendant is now charged with the murder of the victim. He will be able to assert a valid defense if the man's acquittal was based on which of the following?\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}