{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"587cae30-3a51-5699-b9dc-6c0ebeb9bcf0","task_key":"all--test--587cae30-3a51-5699-b9dc-6c0ebeb9bcf0","task_revision_id":"1","upstream_id":"","short_description":"An accountant compiles the financial statements of a nonissuer and issues the…","config":"all","split":"test","body":"{\"choices\":\"['The accountant has not audited or reviewed the financial statements.', 'Substantially all disclosures required by GAAP are included in the financial statements.', 'The financial statements should not be used to obtain credit.', 'The compilation is limited to presenting information that is the representation of management.']\",\"question\":\"An accountant compiles the financial statements of a nonissuer and issues the standard compilation report. Although not specifically stated in this report, it is implied that\",\"subject\":\"professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}