{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9b62390a-8a74-53fd-9541-962a5d7ab590","task_key":"all--test--9b62390a-8a74-53fd-9541-962a5d7ab590","task_revision_id":"1","upstream_id":"","short_description":"Why is there so much uncertainty over which states have nuclear weapons?","config":"all","split":"test","body":"{\"choices\":\"['Leaders have incentives to lie', 'If leaders revealed their programs, they would be more likely to be attacked', 'Leaders will not always grant foreign monitors access to their nuclear programs', 'ALL of the above']\",\"question\":\"Why is there so much uncertainty over which states have nuclear weapons?\",\"subject\":\"us_foreign_policy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}