{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b59db68d-1ed2-5d0a-a343-727d0069336c","task_key":"all--test--b59db68d-1ed2-5d0a-a343-727d0069336c","task_revision_id":"1","upstream_id":"","short_description":"Tina is a very good student: Her grades are high, she is involved in…","config":"all","split":"test","body":"{\"choices\":\"['modeling', 'accommodation', 'a halo effect', 'convergence']\",\"question\":\"Tina is a very good student: Her grades are high, she is involved in extracurricular activities, and typically she excels at anything she tries. Therefore, when she caused an accident her second day of driver's ed., her instructor was shocked. The instructor's disbelief is most probably a result of\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}