{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"00cfa434-c0b4-5581-b95e-f9ed3350dd72","task_key":"all--test--00cfa434-c0b4-5581-b95e-f9ed3350dd72","task_revision_id":"1","upstream_id":"","short_description":"Homo erectus is characterized by:","config":"all","split":"test","body":"{\"choices\":\"['a sophisticated stone-tool technology, cooperative hunting, and the controlled use of fire.', 'a primitive stone-tool technology, the sophisticated use of wood to construct shelters, and the earliest known gardening.', 'cooperative hunting, the controlled use of fire, and elaborate burials with many grave goods.', 'a primitive stone-tool technology, the first evidence for fishing, and the construction of stone buildings.']\",\"question\":\"Homo erectus is characterized by:\",\"subject\":\"prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}