{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d646d608-22ab-517b-89fd-524847627c42","task_key":"all--test--d646d608-22ab-517b-89fd-524847627c42","task_revision_id":"1","upstream_id":"","short_description":"Sociology differs from common sense in that:","config":"all","split":"test","body":"{\"choices\":\"[\\\"it focuses on the researchers' own experiences\\\", 'it makes little distinction between the way the world is and the way it ought to be', 'its knowledge is accumulated from many different research contexts', 'it is subjective and biased']\",\"question\":\"Sociology differs from common sense in that:\",\"subject\":\"sociology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}