{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"72414e37-0803-5cde-8402-f8822815e17f","task_key":"all--test--72414e37-0803-5cde-8402-f8822815e17f","task_revision_id":"1","upstream_id":"","short_description":"Some of the oldest painted art in the world was recently discovered in caves on…","config":"all","split":"test","body":"{\"choices\":\"['animals and human bodies from more than 15,000 to about 20,000 years ago', 'animals and hand stencils from more than 35,000 to about 40,000 years ago', 'animals and hand stencils from more than 45,000 to about 100,000 years ago', 'animals and abstract geometric shapes from 15,000 to more than 100,000 years ago']\",\"question\":\"Some of the oldest painted art in the world was recently discovered in caves on the island of Sulawesi in Indonesia. What do the images include and to when do they date?\",\"subject\":\"prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}