{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1bd7c058-ee0d-5a5b-9bcb-5277837242d2","task_key":"all--test--1bd7c058-ee0d-5a5b-9bcb-5277837242d2","task_revision_id":"1","upstream_id":"","short_description":"Pluto's extremely cold (~40 K) surface is composed of:","config":"all","split":"test","body":"{\"choices\":\"['mainly water ice which always remains frozen', 'nitrogen methane and carbon monoxide ices which sublimate into an atmosphere near perihelion', 'nitrogen methane and carbon monoxide ices which always remain frozen', 'roughly half ices and half rocky materials']\",\"question\":\"Pluto's extremely cold (~40 K) surface is composed of:\",\"subject\":\"astronomy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}