{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d6b27487-08b3-5707-9a50-47d13107da69","task_key":"default--test--7830","task_revision_id":"1","upstream_id":"7830","short_description":"Which of the following is true?","config":"default","split":"test","body":"{\"category\":\"math\",\"options\":\"['Both (a) and (b).', 'Neither (a) nor (b).', 'Every compact space is a metric space', 'Every metric space is compact', 'Neither (a) nor (b) with the exception of metric spaces', 'Every compact space is incomplete', 'Both (a) and (b) with the exception of metric spaces', 'Every complete space is compact', 'Every compact space is complete', 'Every incomplete space is compact']\",\"question\":\"Which of the following is true?\",\"src\":\"ori_mmlu-college_mathematics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}