{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"22667a5d-b6c2-50ed-918d-fda3353b3e08","task_key":"default--test--10295","task_revision_id":"1","upstream_id":"10295","short_description":"You observe the visible spectrum of a star viewed through a cloud of cool…","config":"default","split":"test","body":"{\"category\":\"physics\",\"options\":\"['You see the star’s blackbody spectrum with emission lines due to hydrogen.', 'You see only the star’s blackbody spectrum.', 'You see only emission lines characteristic of helium.', 'You see the star’s blackbody spectrum with no lines due to hydrogen.', 'You see only emission lines characteristic of hydrogen.', 'You see the star’s blackbody spectrum with absorption lines due to helium.', 'You see the star’s blackbody spectrum with absorption lines due to hydrogen.', \\\"You see a mix of the star's blackbody spectrum and emission lines characteristic of hydrogen.\\\", 'You see only absorption lines characteristic of the star’s composition.', 'You see only emission lines characteristic of the star’s composition.']\",\"question\":\"You observe the visible spectrum of a star viewed through a cloud of cool hydrogen gas.\",\"src\":\"ori_mmlu-astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}