{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7934e439-7d6f-55cb-b116-e02ce1913325","task_key":"all--test--7934e439-7d6f-55cb-b116-e02ce1913325","task_revision_id":"1","upstream_id":"","short_description":"Can countries rely on their domestic law as an excuse to violate their…","config":"all","split":"test","body":"{\"choices\":\"['Domestic law always prevails over international law', 'Only customary international law prevails over domestic law', 'Obligations under international law prevail over domestic law', 'Constitutional obligations always prevail over obligations under international law']\",\"question\":\"Can countries rely on their domestic law as an excuse to violate their obligations under international law?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}