{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e2d8cbcf-c80c-5bcc-bfaf-4c479059ca7c","task_key":"default--test--5522","task_revision_id":"1","upstream_id":"5522","short_description":"Children who are still sucking their thumbs when they enter kindergarten are…","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['Formal negative sanction', 'No sanction is applied', 'Positive sanction, but not classified as formal or informal', 'Informal neutral sanction', 'Informal positive sanction', 'Informal negative sanction', 'Negative sanction, but not classified as formal or informal', 'Formal positive sanction', 'Formal neutral sanction', 'Both formal and informal negative sanction']\",\"question\":\"Children who are still sucking their thumbs when they enter kindergarten are often subject to teasing and ridicule for this behavior. What type of social sanction is applied in this instance?\",\"src\":\"ori_mmlu-sociology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}