{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"56870947-d1e7-5066-a22f-38f77aa84fe6","task_key":"all--test--56870947-d1e7-5066-a22f-38f77aa84fe6","task_revision_id":"1","upstream_id":"","short_description":"An audit client has a valid reason for requesting that a certain account…","config":"all","split":"test","body":"{\"choices\":\"[\\\"Verify the account balance by inspecting the client's bank statements and cash receipt records.\\\", 'Select at random a different account for confirmation that is approximately the same size.', \\\"Request the client's management to document the matter in the management representation letter.\\\", 'Explain to the client that the request will most likely cause the auditor to disclaim an opinion.']\",\"question\":\"An audit client has a valid reason for requesting that a certain account receivable that the auditor has selected for confirmation not be confirmed. Under these circumstances the auditor should\",\"subject\":\"professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}