{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f46f81f7-592b-52bd-9e0b-cfec650e3465","task_key":"all--test--f46f81f7-592b-52bd-9e0b-cfec650e3465","task_revision_id":"1","upstream_id":"","short_description":"A program is expressed in a programming language. Which of the following is true…","config":"all","split":"test","body":"{\"choices\":\"['The program can also be expressed as binary code, but will be more easily understood by humans when expressed in a higher-level programming language.', 'The program can also be expressed as binary code, which will reduce the likelihood of errors.', 'The program cannot be expressed as binary code, because binary code can only be used to represent data.', 'Some parts of the program can be expressed as binary code, but operations must be expressed using a higher-level programming language.']\",\"question\":\"A program is expressed in a programming language. Which of the following is true of the program?\",\"subject\":\"high_school_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}