{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fb641d46-4426-5334-8338-8f985a769f8c","task_key":"all--test--fb641d46-4426-5334-8338-8f985a769f8c","task_revision_id":"1","upstream_id":"","short_description":"An auditor is using statistical sampling in testing whether cash disbursements…","config":"all","split":"test","body":"{\"choices\":\"['Planned reliance on the prescribed control.', 'Assessed level of control risk.', 'Planned reliance on substantive tests.', 'Tolerable rate of deviations.']\",\"question\":\"An auditor is using statistical sampling in testing whether cash disbursements were properly authorized. The sample results indicate that the sample rate of deviation plus the allowance for sampling risk exceeds the tolerable rate. Under these circumstances the auditor most likely would reduce the\",\"subject\":\"professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}