{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"58083c16-d58f-5252-b8a7-60c42dcb6169","task_key":"all--test--58083c16-d58f-5252-b8a7-60c42dcb6169","task_revision_id":"1","upstream_id":"","short_description":"Most New Yorkers remember where they were and what they were doing when they…","config":"all","split":"test","body":"{\"choices\":\"['implicit memory', 'engrams', 'flashbulb memory', 'explicit memory']\",\"question\":\"Most New Yorkers remember where they were and what they were doing when they heard that the World Trade Center was destroyed. Which of the following best identifies this type of memory?\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}