{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6188f6a7-9fd4-5580-92db-a12b65d290e1","task_key":"all--test--6188f6a7-9fd4-5580-92db-a12b65d290e1","task_revision_id":"1","upstream_id":"","short_description":"A CPA prepares income tax returns for a client. After the client signs and mails…","config":"all","split":"test","body":"{\"choices\":\"['Document the error in the workpapers.', 'Prepare an amended return within 30 days of the discovery of the error.', 'Promptly advise the client of the error.', 'Promptly resign from the engagement and cooperate with the successor accountant.']\",\"question\":\"A CPA prepares income tax returns for a client. After the client signs and mails the returns, the CPA discovers an error. According to Treasury Circular 230, the CPA must\",\"subject\":\"professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}