{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0d3cbc70-b73e-5928-ad40-8efac7dd22e7","task_key":"default--test--10209","task_revision_id":"1","upstream_id":"10209","short_description":"Our current best observations show that Pluto has","config":"default","split":"test","body":"{\"category\":\"physics\",\"options\":\"['two large satellites and one small satellite.', 'one medium sized satellite and two small satellites.', 'one large satellite and one small satellite.', 'no satellites.', 'one large satellite and two small satellites.', 'four small satellites.', 'three medium sized satellites.', 'two medium sized satellites.', 'one large satellite and three small satellites.', 'one large satellite.']\",\"question\":\"Our current best observations show that Pluto has\",\"src\":\"ori_mmlu-astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}