{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"344fdf76-2c65-589d-94b3-ff57a5ebc87d","task_key":"default--test--4965","task_revision_id":"1","upstream_id":"4965","short_description":"In the author's two-hour movie representing the history of the universe, when do…","config":"default","split":"test","body":"{\"category\":\"history\",\"options\":\"['during the final minute of the film', '90 minutes into the film', 'about half way through the film', 'during the final 30 seconds of the film', 'in the first hour of the film', 'during the final 3 seconds of the film', 'at the end of the film', 'during the final 10 minutes of the film', 'at the beginning of the film', '30 minutes into the film']\",\"question\":\"In the author's two-hour movie representing the history of the universe, when do the earliest members of the human family appear?\",\"src\":\"ori_mmlu-prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}