{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"536de913-27b2-5b59-a075-3375e800979d","task_key":"default--test--5905","task_revision_id":"1","upstream_id":"5905","short_description":"Which of these is not one of Aesop's fables?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"[\\\"The Ox and the Frogs'\\\", \\\"The Hare and the Tortoise'\\\", \\\"The Swan and the Goose'\\\", \\\"The Lion and the Mouse'\\\", \\\"The Eagle and the Beetle'\\\", \\\"The Crow and the Pitcher'\\\", \\\"The Dog and the Squirrel'\\\", \\\"The Hawk and the Nightingale'\\\", \\\"The Cat and the Mice'\\\", \\\"The Fox and the Grapes'\\\"]\",\"question\":\"Which of these is not one of Aesop's fables?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}