{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"5e9bb274-00b8-5e1b-920c-88091a3d0927","task_key":"default--test--9362","task_revision_id":"1","upstream_id":"9362","short_description":"The two moons of Mars are called ...","config":"default","split":"test","body":"{\"category\":\"physics\",\"options\":\"['Deimos and Triton', 'Phobos and Demos', 'Triton and Deimos', 'Phobos and Deimos', 'Tritos and Desmos', 'Phobos and Mars II', 'Phobos and Tritos', 'Phobos and Triton', 'Tritos and Deimos', 'Deimos and Mars I']\",\"question\":\"The two moons of Mars are called ...\",\"src\":\"ori_mmlu-astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}