{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f5fe8f3b-5d8d-5095-bf12-e4251d2eebb7","task_key":"all--test--f5fe8f3b-5d8d-5095-bf12-e4251d2eebb7","task_revision_id":"1","upstream_id":"","short_description":"You have been asked to write a response paper to LeVay's research on brain…","config":"all","split":"test","body":"{\"choices\":\"['Sample size was relatively small.', 'Gay and straight groups were not equivalent in terms of the number of deaths due to AIDS.', 'Lesbians were omitted from the study.', 'all of the above']\",\"question\":\"You have been asked to write a response paper to LeVay's research on brain differences between heterosexuals and homosexuals. Which are plausible criticisms of LeVay's work?\",\"subject\":\"human_sexuality\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}