{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"957f319b-4fda-5cad-b9ff-d575627ffd1c","task_key":"all--test--957f319b-4fda-5cad-b9ff-d575627ffd1c","task_revision_id":"1","upstream_id":"","short_description":"When are countermeasures illegal?","config":"all","split":"test","body":"{\"choices\":\"['Countermeasures are illegal when inter alia are disproportionate or in violation of a peremptory norm of international law (e.g. the use of force, human rights)', 'Countermeasures are illegal when they are taken in the course of an armed conflict', 'Countermeasures are illegal when they violate bilateral treaties', 'Countermeasures are illegal when the responsible State does not consent to them']\",\"question\":\"When are countermeasures illegal?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}