{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e0ffebce-e1f2-5732-a8e7-107d63df3dec","task_key":"all--test--e0ffebce-e1f2-5732-a8e7-107d63df3dec","task_revision_id":"1","upstream_id":"","short_description":"You want to start treating clients who believe they have been abducted by…","config":"all","split":"test","body":"{\"choices\":\"['start seeing these clients and consult with someone who is familiar with this population if you run into any problems.', 'start seeing these clients after establishing a consultative relationship with someone who is familiar with this population.', 'take a continuing education class related to the treatment of members of this population.', 'place an ad in the National Enquirer to let members of this population know about the availability of your services.']\",\"question\":\"You want to start treating clients who believe they have been abducted by aliens, but you have never worked with members of this population before. Your best course of action would be to:\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}