{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b8c86f8b-d2da-527b-9de1-81774739cdec","task_key":"all--test--b8c86f8b-d2da-527b-9de1-81774739cdec","task_revision_id":"1","upstream_id":"","short_description":"The concomitant variation fallacy is","config":"all","split":"test","body":"{\"choices\":\"['attributing the cause of an event to a single cause when the cause is more complicated', 'arguing that since two events happened at the same time, one event must have caused the other', 'arguing that since one event preceded another event it must have caused the second event', 'drawing a conclusion based on sign reasoning when there is not really a direct relationship between the alleged sign and the subject of the conclusion']\",\"question\":\"The concomitant variation fallacy is\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}