{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9791d88e-7bf8-58fc-a5aa-b5357b35cc5c","task_key":"all--test--9791d88e-7bf8-58fc-a5aa-b5357b35cc5c","task_revision_id":"1","upstream_id":"","short_description":"How can customary law be related to treaty provisions?","config":"all","split":"test","body":"{\"choices\":\"['Treaties may only codify customary law', 'Customary law is a different source of international and it cannot be embodied in treaties', \\\"Treaties may 1) codify customary law, 2) 'crystallise' customary law, 3) lead to the emergence of customary law\\\", 'Customary law can emerge only from few multilateral treaties']\",\"question\":\"How can customary law be related to treaty provisions?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}