{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"80a80c84-24a8-5d81-9bca-5cc800a0c133","task_key":"all--test--80a80c84-24a8-5d81-9bca-5cc800a0c133","task_revision_id":"1","upstream_id":"","short_description":"Tonya runs into an old schoolmate on the street. During their brief…","config":"all","split":"test","body":"{\"choices\":\"['tip-of-the-tongue phenomenon', 'chunking', 'deductive reasoning', 'inductive reasoning']\",\"question\":\"Tonya runs into an old schoolmate on the street. During their brief conversation, Tonya is unable to recall the schoolmate's name. Days later, she remembers the name out of the blue. To remember the name \\\"out of the blue\\\" is an example of\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}