{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"86839ee4-5997-5a79-a29e-f81f12de8ff6","task_key":"all--test--86839ee4-5997-5a79-a29e-f81f12de8ff6","task_revision_id":"1","upstream_id":"","short_description":"\" _Ad lazarum_ \" is","config":"all","split":"test","body":"{\"choices\":\"['a specific type of false sign that argues there is necessarily an inverse relationship between cost and quality, so something is necessarily a better value because it costs less', 'arguing that what is true of an entire object must also be true of every individual part of that object', 'a specific type of false analogy that reasons what is true about nature must be true about humans', 'drawing a conclusion without enough evidence to support that conclusion']\",\"question\":\"\\\" _Ad lazarum_ \\\" is\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}