{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6390431f-138a-58c4-a05a-8a7ccab49038","task_key":"default--test--5704","task_revision_id":"1","upstream_id":"5704","short_description":"How many cups of coffee were consumed in the United States in the past week (as…","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['9 billion cups per week', '3 billion cups per week', '10 billion cups per week', '2 billion cups per week', '6 billion cups per week', '300 million cups per week', '500 million cups per week', '4 billion cups per week', '1 billion cups per week', '5 billion cups per week']\",\"question\":\"How many cups of coffee were consumed in the United States in the past week (as of 2013)?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}