{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"5b2955b4-41d0-5642-84e2-407c3a6491ef","task_key":"default--test--11149","task_revision_id":"1","upstream_id":"11149","short_description":"Augustine claims that Academic skepticism:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['is neither true nor false, and cannot be proven.', 'is neither true nor false, but can be refuted.', 'is false and can be refuted.', 'is false but cannot be refuted.', 'is true, and can be proven.', 'is true, but can be refuted.', 'is false, but can be proven.', 'is true, and can be refuted.', 'is false, and cannot be proven.', 'is true, but cannot be proven']\",\"question\":\"Augustine claims that Academic skepticism:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}