{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c25f1a75-1d43-56c7-bed8-4fbe331c087d","task_key":"default--test--7690","task_revision_id":"1","upstream_id":"7690","short_description":"Statement 1 | Every permutation is a cycle. Statement 2 | Every cycle is a…","config":"default","split":"test","body":"{\"category\":\"math\",\"options\":\"['Not enough information, False', 'False, False', 'Not enough information, True', 'True, True', 'True, False', 'True, Not enough information', 'Not enough information, Not enough information', 'False, True', 'False, Not enough information', 'Both statements are partially true']\",\"question\":\"Statement 1 | Every permutation is a cycle. Statement 2 | Every cycle is a permutation.\",\"src\":\"ori_mmlu-abstract_algebra\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}