{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f8a0776c-e9e4-5ecd-8796-68129d74c579","task_key":"all--test--f8a0776c-e9e4-5ecd-8796-68129d74c579","task_revision_id":"1","upstream_id":"","short_description":"A psychologist places an ad in a local newspaper claiming a guaranteed cure for…","config":"all","split":"test","body":"{\"choices\":\"['The ad is not ethical because psychologists are not allowed to advertise', 'The ad is ethical because the psychologist is an excellent psychologist', 'The ad is unethical because it makes misleading claims of effectiveness', 'The ad is ethical because even psychologists are protected by the First Amendment protecting one’s freedom of speech']\",\"question\":\"A psychologist places an ad in a local newspaper claiming a guaranteed cure for anxiety.\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}