{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"14ec290c-2ba5-5103-8846-fbfad5588fdd","task_key":"all--test--14ec290c-2ba5-5103-8846-fbfad5588fdd","task_revision_id":"1","upstream_id":"","short_description":"A programmer is writing a program that is intended to be able to process large…","config":"all","split":"test","body":"{\"choices\":\"['How long the program takes to run', 'How many programming statements the program contains', 'How much memory the program requires as it runs', 'How much storage space the program requires as it runs']\",\"question\":\"A programmer is writing a program that is intended to be able to process large amounts of data. Which of the following considerations is LEAST likely to affect the ability of the program to process larger data sets?\",\"subject\":\"high_school_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}