{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7df64d0c-6b41-5b86-8c30-7ed9a1f02ef2","task_key":"all--test--7df64d0c-6b41-5b86-8c30-7ed9a1f02ef2","task_revision_id":"1","upstream_id":"","short_description":"The argument against the person consists of","config":"all","split":"test","body":"{\"choices\":\"[\\\"ignoring another person's arguments and attacking her character instead\\\", 'asserting an opposing arguer has taken an easily defeated position, which she has not really taken', 'causing confusion during an argument by a lack of ability to engage in refutation', 'using threats of harm instead of reasoning to gain agreement']\",\"question\":\"The argument against the person consists of\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}