{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c74df1a2-024c-58e5-b49f-06e03ffb1744","task_key":"default--test--9217","task_revision_id":"1","upstream_id":"9217","short_description":"When the shadow of the Moon falls on Earth, we have a","config":"default","split":"test","body":"{\"category\":\"physics\",\"options\":\"['lunar eclipse.', \\\"lunar eclipse if it's daytime and solar eclipse if it's nighttime.\\\", 'very dangerous event.', 'planetary alignment.', 'celestial phenomenon.', 'supernova event.', 'solar eclipse if it’s daytime and lunar eclipse if it’s nighttime.', 'solar flare event.', 'solar eclipse.', 'moonlight night.']\",\"question\":\"When the shadow of the Moon falls on Earth, we have a\",\"src\":\"ori_mmlu-conceptual_physics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}