{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bf97387c-7c01-5189-9d8a-e65b58293cde","task_key":"default--test--10811","task_revision_id":"1","upstream_id":"10811","short_description":"Aristotle claims that:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['vice is in our power, but virtue is not.', 'virtue is in our power, and so is vice.', 'vice is in our power, and so is virtue.', 'both virtue and vice are not in our power.', 'virtue is in our power, but vice is not.', 'neither virtue nor vice is in our power.', 'neither virtue nor vice are within our control.', 'virtue is not in our power, but vice is.', 'virtue is in our power, but vice is within our control.', 'vice is in our power, but virtue is within our control.']\",\"question\":\"Aristotle claims that:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}