{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ac817ebc-9eb7-54d8-968b-0ac561628d37","task_key":"all--test--ac817ebc-9eb7-54d8-968b-0ac561628d37","task_revision_id":"1","upstream_id":"","short_description":"A psychologist is appointed by the Court to examine a defendant. The…","config":"all","split":"test","body":"{\"choices\":\"['begin the evaluation by informing the defendant that anything that is discussed maybe talked about in open court', 'begin the evaluation by asking the defendant to indicate information that should be kept confidential', 'refuse to do the evaluation unless the Court agrees thar it will be kept confidential', 'discuss confidentiality only if the defendant asks about it']\",\"question\":\"A psychologist is appointed by the Court to examine a defendant. The psychologist should\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}