{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c5587bb7-7942-5db7-9364-7b754426b86a","task_key":"all--test--c5587bb7-7942-5db7-9364-7b754426b86a","task_revision_id":"1","upstream_id":"","short_description":"Which of the following lists the ingredients of the solar nebula from highest to…","config":"all","split":"test","body":"{\"choices\":\"['hydrogen compounds (H2O CH4 NH3) light gases (H He) metals rocks', 'hydrogen compounds (H2O CH4 NH3) light gases (H He) rocks metals', 'light gases (H He) hydrogen compounds (H2O CH4 NH3) metals rocks', 'light gases (H He) hydrogen compounds (H2O CH4 NH3) rocks metals']\",\"question\":\"Which of the following lists the ingredients of the solar nebula from highest to lowest percentage of mass in the nebula?\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}