{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"af2da289-0878-5e43-911a-9b1b28dc87d9","task_key":"all--test--af2da289-0878-5e43-911a-9b1b28dc87d9","task_revision_id":"1","upstream_id":"","short_description":"Do treaties bind third States, ie non-State parties?","config":"all","split":"test","body":"{\"choices\":\"['Treaties may create only rights for third States', 'Treaties create both obligations and rights for third States', 'Treaties do no create obligations or rights for third States without their consent', 'Treaties do not create any obligations or rights for third States, even when the latter consent.']\",\"question\":\"Do treaties bind third States, ie non-State parties?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}