{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1497ddbc-8a86-5e29-b22d-3f03c85a0199","task_key":"all--test--1497ddbc-8a86-5e29-b22d-3f03c85a0199","task_revision_id":"1","upstream_id":"","short_description":"A ten-year-old boy, tested with a paper-and-pencil intelligence test, obtained…","config":"all","split":"test","body":"{\"choices\":\"['more than 130', 'less than 130', '130', '130, but with a different pare of subtest scores']\",\"question\":\"A ten-year-old boy, tested with a paper-and-pencil intelligence test, obtained an IQ score of 130. Two years later, he was tested with an alternate form of the same test. Most likely, the obtained score on the second testing would be\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}