{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"12096765-1aba-5372-8f8f-7c80cadbe78d","task_key":"all--test--12096765-1aba-5372-8f8f-7c80cadbe78d","task_revision_id":"1","upstream_id":"","short_description":"\" _Ad crumenam_ \" is","config":"all","split":"test","body":"{\"choices\":\"['basing a conclusion on an analogy when the items being compared are not similar enough to sustain the analogy', 'a specific type of false sign fallacy that reasons someone is necessarily better because he or she is wealthy, or is worse because he or she is poor', 'drawing a firm conclusion without enough evidence to support that conclusion', 'a specific type of false cause that involves mistakenly attributing the cause of an event to a simple cause when the cause is more complex']\",\"question\":\"\\\" _Ad crumenam_ \\\" is\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}