{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"13f8d397-5c92-59ee-8bef-d42e238d7899","task_key":"all--test--13f8d397-5c92-59ee-8bef-d42e238d7899","task_revision_id":"1","upstream_id":"","short_description":"What is a tracheostomy?","config":"all","split":"test","body":"{\"choices\":\"['An opening in the anterior wall of the trachea above the cricoid cartilage.', 'An opening in the anterior wall of the trachea below the cricoid cartilage.', 'An opening in the anterior chest wall.', 'An opening in the posterior wall of the trachea below the cricoid cartilage.']\",\"question\":\"What is a tracheostomy?\",\"subject\":\"clinical_knowledge\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}