{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1b4c87f5-1c6e-5ee2-a3d7-3b6d36d7a7ea","task_key":"default--test--10885","task_revision_id":"1","upstream_id":"10885","short_description":"Appeal to compassion is","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['an appeal to emotion that argues a conclusion should be made based only on feeling sorry for someone', 'an appeal to emotion that argues a conclusion should be made based solely on feeling disgusted by someone', 'an appeal to emotion that argues a conclusion should be made based solely on fear', 'arguing that an idea should be rejected based only on who originated the idea', 'arguing that an idea should be accepted based only on who originated the idea', 'an appeal to emotion that argues a conclusion should be made based solely on feeling happy for someone', 'an appeal to emotion that argues an action should be taken based only on the need to be loyal to someone', 'an appeal to emotion that argues an action should be taken based solely on anger towards someone', 'arguing that an idea should be accepted based solely on the popularity of the idea', 'an appeal to emotion that argues against a position based only on negative personal feelings toward the position']\",\"question\":\"Appeal to compassion is\",\"src\":\"ori_mmlu-logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}