{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"96cc0f31-9670-5bf8-8f9a-114f06e40112","task_key":"all--test--96cc0f31-9670-5bf8-8f9a-114f06e40112","task_revision_id":"1","upstream_id":"","short_description":"Dr. Pavel Petrovich, a licensed psychologist, has been seeing a client in…","config":"all","split":"test","body":"{\"choices\":\"['advise the client that he is ethically obligated to terminate therapy.', 'raise for discussion the possibility of termination and referral to another therapist.', 'search the literature for discussions of similar cases that have been treated and reported by others.', 'continue seeing the client since fourteen months is not an unusually long time for therapy to continue without a significant change in symptoms.']\",\"question\":\"Dr. Pavel Petrovich, a licensed psychologist, has been seeing a client in therapy for over fourteen months with no apparent change in the client's symptoms. Dr. Petrovich should:\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}