{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8866931a-5354-5c96-8893-dd3242931453","task_key":"default--test--7966","task_revision_id":"1","upstream_id":"7966","short_description":"30 students from 5 classes solved 40 math problems. Each student must answer at…","config":"default","split":"test","body":"{\"category\":\"math\",\"options\":\"['22', '24', '25', '30', '26', '18', '20', '28', '15', '10']\",\"question\":\"30 students from 5 classes solved 40 math problems. Each student must answer at least one question. Every two students in the same class solved the same number of questions. The number of questions answered by any two students in different classes is also different. Question: What's maximum possible number of students who only answered one question?\",\"src\":\"theoremQA-Math\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}