{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"508be89a-14e0-500a-a76c-ced7c53bb0cf","task_key":"all--test--508be89a-14e0-500a-a76c-ced7c53bb0cf","task_revision_id":"1","upstream_id":"","short_description":"The lithosphere of a planet is the layer that consists of","config":"all","split":"test","body":"{\"choices\":\"['the softer rocky material of the mantle.', 'the lava that comes out of volcanoes.', 'material between the crust and the mantle.', 'the rigid rocky material of the crust and uppermost portion of the mantle.']\",\"question\":\"The lithosphere of a planet is the layer that consists of\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}