{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"16ff7f06-6241-55a3-a577-57dce72edc2f","task_key":"default--test--7049","task_revision_id":"1","upstream_id":"7049","short_description":"Which of the following will increase wages for tuba makers?","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['A decrease in the cost of tuba-making materials', 'An increase in the number of graduates at tuba maker training school', 'An increase in the tax on tubas', 'An increase in the number of tuba maker training schools', 'An increase in the price of tubas']\",\"question\":\"Which of the following will increase wages for tuba makers?\",\"src\":\"ori_mmlu-high_school_microeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}