{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c87854d5-499a-521b-b1f0-53c304b3cb62","task_key":"default--test--5282","task_revision_id":"1","upstream_id":"5282","short_description":"What is direct diplomacy?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['Diplomats negotiating on behalf of state leaders.', 'The president consulting Congress on foreign policy issues', 'Multilateral talks involving several nations', 'State leaders communicating through written letters', 'Use of social media by state leaders to address foreign policy issues', 'Members of Congress negotiating directly with foreign governments', 'Foreign policy decisions made without consulting with other nations', 'Non-governmental organizations conducting negotiations with foreign entities']\",\"question\":\"What is direct diplomacy?\",\"src\":\"ori_mmlu-us_foreign_policy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}