{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4ddcfc6b-0ccf-5b20-a751-1782754e589d","task_key":"default--test--9476","task_revision_id":"1","upstream_id":"9476","short_description":"What is true for a type-Ia (\"type one-a\") supernova?","config":"default","split":"test","body":"{\"category\":\"physics\",\"options\":\"['This type occurs in binary systems.', 'This type produces high amounts of X-rays.', 'This type produces no observable light.', 'This type occurs in old galaxies.', 'This type produces high amounts of infrared light.', 'This type produces high amounts of ultraviolet light.', 'This type occurs only in our galaxy.', 'This type occurs in isolated stars.', 'This type occurs in young galaxies.', 'This type produces gamma-ray bursts.']\",\"question\":\"What is true for a type-Ia (\\\"type one-a\\\") supernova?\",\"src\":\"ori_mmlu-astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}