{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"42220704-34ae-5b9d-ba67-de382d99bd13","task_key":"all--test--42220704-34ae-5b9d-ba67-de382d99bd13","task_revision_id":"1","upstream_id":"","short_description":"Which of the following best describes the snob appeal fallacy?","config":"all","split":"test","body":"{\"choices\":\"['reasoning that a claim is true based only on the evidence that it has been made so often.', 'arguing someone should accept a claim based only on the evidence that he or she has the favorable characteristics to understand the truth of the claim.', 'arguing a claim should be accepted based only on evidence that unidentified authorities accept it.', 'arguing that someone should accept an idea or product based only on evidence that prestigious people accept it, or that non-prestigious people reject it.']\",\"question\":\"Which of the following best describes the snob appeal fallacy?\",\"subject\":\"logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}