{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"61361379-a6b9-59db-83bf-9b985918df1b","task_key":"all--test--61361379-a6b9-59db-83bf-9b985918df1b","task_revision_id":"1","upstream_id":"","short_description":"A large Java program was tested extensively, and no errors were found. What can…","config":"all","split":"test","body":"{\"choices\":\"['All of the preconditions in the program are correct.', 'All of the postconditions in the program are correct.', 'The program may have bugs.', 'Every method in the program may safely be used in other programs.']\",\"question\":\"A large Java program was tested extensively, and no errors were found. What can be concluded?\",\"subject\":\"high_school_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}