{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"99f31778-2a67-542b-b338-5ded45473470","task_key":"all--test--99f31778-2a67-542b-b338-5ded45473470","task_revision_id":"1","upstream_id":"","short_description":"Dr. A. Praise is hired to perform a court-ordered evaluation of a defendant in a…","config":"all","split":"test","body":"{\"choices\":\"['remind the defendant that he has no choice since the evaluation is court-ordered.', 'refuse to conduct the evaluation unless she obtains a signed consent from the defendant.', 'refuse to conduct the evaluation unless she obtains a signed consent from the defendants attorney.', 'postpone the evaluation and advise the defendant to contact his attorney.']\",\"question\":\"Dr. A. Praise is hired to perform a court-ordered evaluation of a defendant in a criminal case. She discusses the purpose of the evaluation with the defendant who then says he does not want to cooperate. According to the Specialty Guidelines for Forensic Psychology, Dr. Praise should:\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}