{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3252298e-87d9-5b21-9e5e-d89e3df83b52","task_key":"default--test--4870","task_revision_id":"1","upstream_id":"4870","short_description":"Homo erectus was:","config":"default","split":"test","body":"{\"category\":\"history\",\"options\":\"['a stable, long-lived species.', 'the first hominid to stand “erect” and walk upright.', 'a hominid species that was completely herbivorous.', 'the first hominid to develop agriculture.', 'an unsuccessful, short-lived species.', 'a hominid species that lived primarily in trees.', 'a species that coexisted with modern humans.', 'the first hominid to develop complex language.', 'the first hominid to make and use stone tools.']\",\"question\":\"Homo erectus was:\",\"src\":\"ori_mmlu-prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}