{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7818b5fb-c9e8-5af6-b97f-6dd4cfec260b","task_key":"all--test--7818b5fb-c9e8-5af6-b97f-6dd4cfec260b","task_revision_id":"1","upstream_id":"","short_description":"Which of the following defines a Revolution in Military Affairs?","config":"all","split":"test","body":"{\"choices\":\"['A coup in which established leaders are ousted by an internal military force.', 'A wholesale shift in the way warfare is conducted and understood.', 'A watershed in technological advancement, such as the creation of nuclear weaponry.', 'A watershed in technological advancement which alters the fundamental characteristics of war.']\",\"question\":\"Which of the following defines a Revolution in Military Affairs?\",\"subject\":\"security_studies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}