{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4e10b01b-35ed-5ee8-9ca3-51da3557ef66","task_key":"all--test--4e10b01b-35ed-5ee8-9ca3-51da3557ef66","task_revision_id":"1","upstream_id":"","short_description":"What kind of actors are involved in the defence trade?","config":"all","split":"test","body":"{\"choices\":\"['The defence trade concerns legitimate business interaction between states procuring or selling military equipment.', 'Military organisations are the sole actors in the defence trade.', 'Not only state actors are involved in the defence trade. Non-state actors, including illegal groups, are also key players.', 'The defence trade refers solely to the trade in weapons and small arms that takes place outside the legal interactions between states.']\",\"question\":\"What kind of actors are involved in the defence trade?\",\"subject\":\"security_studies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}