{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e64b89a2-7b0d-59ac-aee2-fe35d06b0aeb","task_key":"all--test--e64b89a2-7b0d-59ac-aee2-fe35d06b0aeb","task_revision_id":"1","upstream_id":"","short_description":"Bob writes down a number between 1 and 1,000. Mary must identify that number by…","config":"all","split":"test","body":"{\"choices\":\"['1,000', '999', '500', '10']\",\"question\":\"Bob writes down a number between 1 and 1,000. Mary must identify that number by asking \\\"yes/no\\\" questions of Bob. Mary knows that Bob always tells the truth. If Mary uses an optimal strategy, then she will determine the answer at the end of exactly how many questions in the worst case?\",\"subject\":\"college_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}