{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"24b1c310-3d3a-5559-8db0-b34f52596090","task_key":"default--test--4737","task_revision_id":"1","upstream_id":"4737","short_description":"There is evidence that premodern Homo sapiens existed:","config":"default","split":"test","body":"{\"category\":\"history\",\"options\":\"['500,000 to 100,000 years ago.', '800,000 to 200,000 years ago.', '200,000 to 50,000 years ago.', '700,000 to 40,000 years ago.', '600,000 to 30,000 years ago.', '100,000 to 10,000 years ago.', '1,000,000 to 500,000 years ago.', '900,000 to 10,000 years ago.', '400,000 to 10,000 years ago.', '300,000 to 20,000 years ago.']\",\"question\":\"There is evidence that premodern Homo sapiens existed:\",\"src\":\"ori_mmlu-prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}