{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a69e0746-1e32-5355-bee6-934b5e26f476","task_key":"default--test--7505","task_revision_id":"1","upstream_id":"7505","short_description":"If one person has the only original signed copy of The Wealth of Nations by Adam…","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['A downward sloping supply curve.', 'An upward sloping demand curve.', 'An upward sloping supply curve.', 'An invisible hand.', 'A perfectly horizontal supply curve.', 'A perfectly horizontal demand curve.', 'A perfectly vertical demand curve.', 'An invisible supply.', 'A downward sloping demand curve.', 'A perfectly vertical supply curve.']\",\"question\":\"If one person has the only original signed copy of The Wealth of Nations by Adam Smith, which of the following would illustrate this situation?\",\"src\":\"ori_mmlu-high_school_microeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}