{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fbb08bf8-26aa-5cff-89c2-c43d1b4dcd6d","task_key":"all--test--fbb08bf8-26aa-5cff-89c2-c43d1b4dcd6d","task_revision_id":"1","upstream_id":"","short_description":"An evaluation of a particular treatment fails to show an overall effect.…","config":"all","split":"test","body":"{\"choices\":\"['is valid if t test comparing severely disturbed patients with less severely disturbed patients is significant', 'fails to take into account regression toward the mean', 'fails to take into account historical and maturational variables', 'fails to take into account the fact thar the “placebo effect” is most pronounced in severely disturbed patients']\",\"question\":\"An evaluation of a particular treatment fails to show an overall effect. However, the experimenter does find thar the most severely disturbed do show considerable improvement, and therefore concludes that the treatment is effective with extreme cases. This conclusion\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}