{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"57458d75-19b2-5a64-b520-c03c5783331a","task_key":"all--test--57458d75-19b2-5a64-b520-c03c5783331a","task_revision_id":"1","upstream_id":"","short_description":"Marcia Baron argues that the artificiality of ticking bomb cases is problematic…","config":"all","split":"test","body":"{\"choices\":\"['the artificial assumptions are, in fact, empirical claims that are central to the debate.', 'all artificial thought experiments are philosophically useless.', 'it leads us to believe that torture is always permissible.', 'all of the above']\",\"question\":\" Marcia Baron argues that the artificiality of ticking bomb cases is problematic because\",\"subject\":\"moral_disputes\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}