{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b1873218-c3d0-593c-965b-6756a95beba9","task_key":"default--test--5824","task_revision_id":"1","upstream_id":"5824","short_description":"What is the approximate speed of light?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['165 miles per hour', '122000 miles per hour', '983571056 miles per hour', '186000 miles per second', '186282 miles per hour', '700000 miles per second', '293000 miles per second', '186000 miles per hour', '300000 miles per second', '670616629 miles per hour']\",\"question\":\"What is the approximate speed of light?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}