{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"04ad77b0-d194-5978-99d4-26fef027debd","task_key":"all--test--04ad77b0-d194-5978-99d4-26fef027debd","task_revision_id":"1","upstream_id":"","short_description":"Are States under any obligation when they conduct negotiations?","config":"all","split":"test","body":"{\"choices\":\"['States are under no obligation as to how they conduct their negotiation', 'States are under an obligation so to conduct themselves that the negotiations are meaningful', 'States are under an obligation to find a solution of the dispute whenever they conduct negotiations', 'States are under an obligation to inform the UN Secretary-General about the progress of their negotiations']\",\"question\":\"Are States under any obligation when they conduct negotiations?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}