{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0e663ad7-7072-519c-b314-8b10a66ddcd0","task_key":"all--test--0e663ad7-7072-519c-b314-8b10a66ddcd0","task_revision_id":"1","upstream_id":"","short_description":"Why do we look for water-ice in craters at Mercury's pole?","config":"all","split":"test","body":"{\"choices\":\"['Actually water-ice is all over Mercury and not just at the poles.', 'The pole is the only place fortunate enough to have had comet impacts', \\\"Radar from the earth can only see Mercury's poles.\\\", 'These craters contain the only permanently shadowed regions on Mercury']\",\"question\":\"Why do we look for water-ice in craters at Mercury's pole?\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}