{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"21c22997-bf00-5297-aef3-af0cdb2c674b","task_key":"all--test--21c22997-bf00-5297-aef3-af0cdb2c674b","task_revision_id":"1","upstream_id":"","short_description":"You are a psychologist in a state/province that grants the right of privileged…","config":"all","split":"test","body":"{\"choices\":\"['Refuse to testify unless you get written permission from your clients', 'Testify', 'Make a determination as to whether or not your testimony would be useful to your client, and testify if you think it would help', 'Testify only if you are granted immunity from litigation']\",\"question\":\"You are a psychologist in a state/province that grants the right of privileged communication to clients of psychologist. The attorney of the spouse of a client you are seeing requests that you testify in their divorce hearing. What is the most appropriate course of action if divorce is not specifically covered under the privilege law\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}