{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"164ee66d-e761-5634-a21d-b48500691a28","task_key":"all--test--164ee66d-e761-5634-a21d-b48500691a28","task_revision_id":"1","upstream_id":"","short_description":"Of the following potential benefits, which is LEAST likely to be provided by the…","config":"all","split":"test","body":"{\"choices\":\"['Human representatives will not be needed to respond to some inquiries.', 'The company will be able to provide a human representative for any incoming call.', 'Customers are likely to spend less time listening to information not relevant to their issue.', 'Customers will be unable to mistakenly select the incorrect department for their particular issue.']\",\"question\":\"Of the following potential benefits, which is LEAST likely to be provided by the upgraded system?\",\"subject\":\"high_school_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}