{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"60f1f9c2-cffb-5a09-be23-b3c0ae5626a6","task_key":"all--test--60f1f9c2-cffb-5a09-be23-b3c0ae5626a6","task_revision_id":"1","upstream_id":"","short_description":"What is a 'treaty' according to the Vienna Convention on the Law of Treaties…","config":"all","split":"test","body":"{\"choices\":\"['Treaties are all agreements concluded between States, international organizations and non-State entities (e.g. corporations)', 'Treaties are agreements concluded between States in written form and governed by international law', 'Treaties are both the written and oral agreements between States', 'Treaties are agreements concluded between States in written form governed either by international or domestic law']\",\"question\":\"What is a 'treaty' according to the Vienna Convention on the Law of Treaties (VCLT)?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}