{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bbc03c9f-6f90-571b-863c-8a1545d397ef","task_key":"default--test--5463","task_revision_id":"1","upstream_id":"5463","short_description":"At the equator how fast is the earth's surface turning?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['about 2500 miles per hour', 'about 3000 miles per hour', 'about 3500 miles per hour', 'about 100 miles per hour', 'about 2000 miles per hour', 'about 500 miles per hour', 'about 1500 miles per hour', 'about 750 miles per hour', 'about 1000 miles per hour', 'about 200 miles per hour']\",\"question\":\"At the equator how fast is the earth's surface turning?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}