{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a896ef3f-e45e-57fa-9ac8-a976cb14713d","task_key":"all--test--a896ef3f-e45e-57fa-9ac8-a976cb14713d","task_revision_id":"1","upstream_id":"","short_description":"A committee of middle school teachers is meeting to begin the process of finding…","config":"all","split":"test","body":"{\"choices\":\"['Downloading a trial version of the most popular software for evaluation', 'Conducting a survey to determine the importance of drafting skills in the classroom', 'Defining the learning objectives they would like the software to meet', 'Deciding what type of user interface would be most appropriate']\",\"question\":\"A committee of middle school teachers is meeting to begin the process of finding a software program to improve students' drafting skills. Which of the following steps should the committee take first in the process?\",\"subject\":\"miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}