{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a34bd4de-4041-5be9-b68a-742cbc48a189","task_key":"all--test--a34bd4de-4041-5be9-b68a-742cbc48a189","task_revision_id":"1","upstream_id":"","short_description":"The Standard for Educational and Psychological Testing provides","config":"all","split":"test","body":"{\"choices\":\"['reliability and validity data on over 3000 tests, along with minimum training requirements needed to administer, score, and interpret each test', 'derailed legal requirements for the use of tests and manuals', 'criteria by which tests and manuals should be judged, and which tests should strive to meet', 'lists of approved tests and manuals']\",\"question\":\"The Standard for Educational and Psychological Testing provides\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}