{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d2cf54ba-062e-556d-8315-d828303814bd","task_key":"all--test--d2cf54ba-062e-556d-8315-d828303814bd","task_revision_id":"1","upstream_id":"","short_description":"What is the role of the compiler in the process of developing executable…","config":"all","split":"test","body":"{\"choices\":\"['Managing specification files created as part of the development process', 'Running and testing the executable created by the programmer', 'Tracking older versions of the software in case an error is found and the software needs to be reverted to an earlier form', 'Translating a program written in an abstract, high-level language into a program with the same behavior expressed in machine code']\",\"question\":\"What is the role of the compiler in the process of developing executable software?\",\"subject\":\"high_school_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}