{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b5f2be43-4624-5092-8cc0-286f7b0f7757","task_key":"all--test--b5f2be43-4624-5092-8cc0-286f7b0f7757","task_revision_id":"1","upstream_id":"","short_description":"You observe the visible spectrum of a star viewed through a cloud of cool…","config":"all","split":"test","body":"{\"choices\":\"['You see only the star’s blackbody spectrum.', 'You see the star’s blackbody spectrum with absorption lines due to hydrogen.', 'You see only emission lines characteristic of hydrogen.', 'You see only emission lines characteristic of the star’s composition.']\",\"question\":\"You observe the visible spectrum of a star viewed through a cloud of cool hydrogen gas.\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}