{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0357f84f-922b-530d-b00e-91f0adf59270","task_key":"default--test--5133","task_revision_id":"1","upstream_id":"5133","short_description":"The Strategic Arms Reduction Treaty was the first accord","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['mandating the elimination of chemical weapons.', 'focused on reducing short-range nuclear weapons.', 'promoting the denuclearization of North Korea.', 'focused on reducing nuclear submarines.', 'on nuclear weapons signed between the United States and the Soviet Union.', 'mandating the elimination of many long-range nuclear missiles.', 'cutting conventional arms in Europe.', 'signed between the United States and China.', 'to be rejected by the U.S. Senate.', 'that allowed the inspection of military bases.']\",\"question\":\"The Strategic Arms Reduction Treaty was the first accord\",\"src\":\"ori_mmlu-us_foreign_policy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}