{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"47b84d47-3e87-5aa6-bb71-62d05cf63259","task_key":"all--test--47b84d47-3e87-5aa6-bb71-62d05cf63259","task_revision_id":"1","upstream_id":"","short_description":"Which of the following steps in the design of a dog house would be earliest?","config":"all","split":"test","body":"{\"choices\":\"['Acquiring all the tools necessary for the building', 'Purchasing all the materials for building the structure', 'Identifying the color, shape, and style of the exterior surface', 'Accessing the measurements of the dog and all the interior materials']\",\"question\":\"Which of the following steps in the design of a dog house would be earliest?\",\"subject\":\"miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}