{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"67624eb0-09e8-5b47-a599-cf7ea5d5d27e","task_key":"all--test--67624eb0-09e8-5b47-a599-cf7ea5d5d27e","task_revision_id":"1","upstream_id":"","short_description":"While best known for protecting freedom of speech and religion, the First…","config":"all","split":"test","body":"{\"choices\":\"['due process of the law in any criminal case', 'retain personal property unless justly compensated by the government', 'not be subjected to excessive fines or unusual punishment', 'petition the government for a redress of grievances']\",\"question\":\"While best known for protecting freedom of speech and religion, the First Amendment also protects Americans' right to\",\"subject\":\"high_school_government_and_politics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}