{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9a64bb88-a154-5092-ac8a-0516b3372df2","task_key":"all--test--9a64bb88-a154-5092-ac8a-0516b3372df2","task_revision_id":"1","upstream_id":"","short_description":"Shapiro discusses experiments about the role of expectations in the experience…","config":"all","split":"test","body":"{\"choices\":\"['drug effects are inherently biological and are not influenced at all by expectations and other psychological factors.', 'users who do not expect to get addicted to \\\"hard\\\" drugs almost always do.', 'there is no such thing as drug addiction.', \\\"sometimes a drug's effects are the direct result of what one expects from the drug.\\\"]\",\"question\":\" Shapiro discusses experiments about the role of expectations in the experience of drug use to show that\",\"subject\":\"moral_disputes\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}