{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e130dc47-6e93-5844-894e-d34ac2a0b512","task_key":"all--test--e130dc47-6e93-5844-894e-d34ac2a0b512","task_revision_id":"1","upstream_id":"","short_description":"Brown cosigned Royal's $50000 note to State Bank. If Royal is later adjudicated…","config":"all","split":"test","body":"{\"choices\":\"['Liable to pay State on the due date of the note.', 'Liable to pay State only if State first seeks payment from Royal.', \\\"Not liable to pay State because Royal's incompetency discharges Royal as a surety.\\\", 'Not liable to pay State unless Brown was a compensated surety.']\",\"question\":\"Brown cosigned Royal's $50000 note to State Bank. If Royal is later adjudicated mentally incompetent what would be Brown's liability on the note?\",\"subject\":\"professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}