{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7eff1959-77d1-5799-a821-a0db98839ab7","task_key":"all--test--7eff1959-77d1-5799-a821-a0db98839ab7","task_revision_id":"1","upstream_id":"","short_description":"Donia was soaked by an unexpected cloudburst while walking to her car from the…","config":"all","split":"test","body":"{\"choices\":\"['confirmation bias', 'limited visualization', 'functional fixedness', 'conceptual constriction']\",\"question\":\"Donia was soaked by an unexpected cloudburst while walking to her car from the office. The fact that she failed to realize that the newspaper she was carrying would have made a great makeshift umbrella is an example of\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}