{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f71cb4dc-4b50-5d9d-b26d-abf78243286a","task_key":"all--test--f71cb4dc-4b50-5d9d-b26d-abf78243286a","task_revision_id":"1","upstream_id":"","short_description":"When a reservation is considered as invalid under the law of treaties?","config":"all","split":"test","body":"{\"choices\":\"['A reservation is invalid when the majority of the State parties objects to it', 'A reservation is invalid only when an international tribunal says so', 'A reservation is invalid only when is incompatible with a peremptory norm of international law (jus cogens)', 'A reservation is invalid when it is incompatible with the object and purpose of the treaty']\",\"question\":\"When a reservation is considered as invalid under the law of treaties?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}