{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b0ba0393-6656-52a7-ac84-de1417da6c4d","task_key":"all--test--b0ba0393-6656-52a7-ac84-de1417da6c4d","task_revision_id":"1","upstream_id":"","short_description":"Why is Catalhoyuk in central Turkey regarded as a large village but not a city?","config":"all","split":"test","body":"{\"choices\":\"['There were no palaces, public architecture, and little evidence for status differentiation.', 'There was no evidence for food storage, domesticated animals, or buildings.', 'There were no works of art, rituals, or activities associated with cities.', 'There was no evidence for monumental architecture or roads.']\",\"question\":\"Why is Catalhoyuk in central Turkey regarded as a large village but not a city?\",\"subject\":\"prehistory\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}