{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"95fc91b8-94d6-5d8e-a352-39f7f097644b","task_key":"default--test--10685","task_revision_id":"1","upstream_id":"10685","short_description":"Bob writes down a number between 1 and 1,000. Mary must identify that number by…","config":"default","split":"test","body":"{\"category\":\"computer science\",\"options\":\"['250', '20', '500', '100', '2', '1,000', '999', '50', '10']\",\"question\":\"Bob writes down a number between 1 and 1,000. Mary must identify that number by asking \\\"yes/no\\\" questions of Bob. Mary knows that Bob always tells the truth. If Mary uses an optimal strategy, then she will determine the answer at the end of exactly how many questions in the worst case?\",\"src\":\"ori_mmlu-college_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}