{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"77e0133d-aed3-5d5e-a3ff-cc93dc2b842f","task_key":"all--test--77e0133d-aed3-5d5e-a3ff-cc93dc2b842f","task_revision_id":"1","upstream_id":"","short_description":"When will the next major impact occur on Earth?","config":"all","split":"test","body":"{\"choices\":\"['Hundreds of millennia in the future.', 'Major impacts can no longer occur since the period of heavy bombardment is over.', 'Hundreds of millennia after the last major impact.', 'Could be any time. The probability of impact is the same next year as it is for any later year.']\",\"question\":\"When will the next major impact occur on Earth?\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}