{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"02d81485-2cb4-5ecf-a18d-cc8d79b9eaff","task_key":"default--test--5725","task_revision_id":"1","upstream_id":"5725","short_description":"The president's principal civilian adviser on the military is the","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['secretary of the treasury.', 'chief of staff.', 'secretary of defense.', 'secretary of state.', 'attorney general.', 'secretary of veterans affairs.', 'secretary of homeland security.', 'vice president.', 'national security adviser.', 'director of the CIA.']\",\"question\":\"The president's principal civilian adviser on the military is the\",\"src\":\"ori_mmlu-us_foreign_policy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}