{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bcd2b63c-39f3-571a-b107-67d22609c843","task_key":"all--test--bcd2b63c-39f3-571a-b107-67d22609c843","task_revision_id":"1","upstream_id":"","short_description":"Hillary just found out that she is pregnant but she suspects that her husband…","config":"all","split":"test","body":"{\"choices\":\"['tubal pregnancies', 'premature delivery', 'spontaneous abortion', 'all of the above']\",\"question\":\"Hillary just found out that she is pregnant but she suspects that her husband has been fooling around. It is important that she be tested for chlamydia because chlamydia can cause which of the following complications?\",\"subject\":\"human_sexuality\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}