{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"edd33353-01ad-5b4b-ab4a-7bbbe8fb30c0","task_key":"default--test--7740","task_revision_id":"1","upstream_id":"7740","short_description":"Nine bags of bird feed are in the storage room. Seventeen more bags will be…","config":"default","split":"test","body":"{\"category\":\"math\",\"options\":\"['60', '100', '120', '80', '90', '110', '25', '9', '45', '70']\",\"question\":\"Nine bags of bird feed are in the storage room. Seventeen more bags will be delivered on Monday. Twenty-two bags will be delivered on Tuesday. Three bags will be delivered on Wednesday. Eleven bags will be delivered on Thursday. Lastly, eighteen bags will be delivered on Friday. By the end of the week, how many bags of bird feed will there be in total?\",\"src\":\"ori_mmlu-high_school_mathematics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}