{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fb46193f-b54b-5067-b4e2-76942adaa173","task_key":"all--test--fb46193f-b54b-5067-b4e2-76942adaa173","task_revision_id":"1","upstream_id":"","short_description":"A psychologist decides to use a computerized test service to facilitate scoring…","config":"all","split":"test","body":"{\"choices\":\"['the use of computerized interpretations is prohibited by ethical guidelines.', 'computerized scoring and interpretation services are notoriously unreliable.', 'computerized interpretations should always be supplemented with other information obtained by the psychologist.', 'because of their objectivity, computerized interpretations are preferable to subjective interpretations.']\",\"question\":\"A psychologist decides to use a computerized test service to facilitate scoring and interpretation of the MMPI-2, which he frequently administers to his clients. The psychologist should be aware that:\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}