{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"91028438-423c-5194-8719-316cd42f707d","task_key":"all--test--91028438-423c-5194-8719-316cd42f707d","task_revision_id":"1","upstream_id":"","short_description":"A student in the process of solving a fabrication problem in the manufacturing…","config":"all","split":"test","body":"{\"choices\":\"['give an opinion as to the best assembly procedure for the particular problem', 'suggest two or three possible assembly procedures and have the student select one', 'place the responsibility completely on the student for making the judgment', 'use leading questions to help the student review and analyze the relative merits of several assembly procedures']\",\"question\":\"A student in the process of solving a fabrication problem in the manufacturing laboratory asks the teacher what assembly procedures should be used. The teacher's best response would be to\",\"subject\":\"miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}