{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4eabe75c-056e-5b53-b22a-7a7f39a9b2de","task_key":"default--test--5521","task_revision_id":"1","upstream_id":"5521","short_description":"Smith & Tomlinson argued that:","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['differences in ethnic background were more significant than class or gender in predicting educational achievement', 'social class was the most significant factor in determining educational success', 'the move towards equality of opportunity for children from ethnic minorities was not undermined by racism in schools', 'inequalities of class had no impact on gender differences in educational achievement', 'ethnic background was less significant than class or gender in predicting educational achievement', 'racism in schools undermined the move towards equality of opportunity for children from ethnic minorites', 'gender and ethnic background had no impact on educational achievement', 'ethnic background and class were equally significant in predicting educational achievement', 'gender differences in educational achievement could be attributed to inequalities of class', 'school character far outweighed ethnic background in determining educational success']\",\"question\":\"Smith & Tomlinson argued that:\",\"src\":\"ori_mmlu-sociology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}