{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e5090930-7f3b-5f24-903f-84c7171c8d6d","task_key":"all--test--e5090930-7f3b-5f24-903f-84c7171c8d6d","task_revision_id":"1","upstream_id":"","short_description":"Which of the following is true as of 2020?","config":"all","split":"test","body":"{\"choices\":\"['males are more likely to have depression and are more likely to have an alcohol use disorder than females.', 'females are more likely to have depression and are more likely to have an alcohol use disorder than males.', 'males are more likely than females to have depression while females are more likely than males to have an alcohol use disorder.', 'females are more likely than males to have depression while males are more likely than females to have an alcohol use disorder.']\",\"question\":\"Which of the following is true as of 2020?\",\"subject\":\"global_facts\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}