{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d9633452-d2bb-5870-b261-6bf21396a310","task_key":"all--test--d9633452-d2bb-5870-b261-6bf21396a310","task_revision_id":"1","upstream_id":"","short_description":"Appeal to compassion is","config":"all","split":"test","body":"{\"choices\":\"['an appeal to emotion that argues against a position based only on negative personal feelings toward the position', 'an appeal to emotion that argues a conclusion should be made based only on feeling sorry for someone', 'an appeal to emotion that argues an action should be taken based only on the need to be loyal to someone', 'arguing that an idea should be accepted based only on who originated the idea']\",\"question\":\"Appeal to compassion is\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}