{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f69e7085-7840-5875-add4-1b58c11e8779","task_key":"default--test--11068","task_revision_id":"1","upstream_id":"11068","short_description":"Feinberg claims that the story about Abraham Lincoln:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['provides some evidence against psychological egoism.', 'supports the theory of psychological altruism.', 'is irrelevant to the discussion of psychological egoism.', 'supports the idea of ethical egoism.', 'proves the concept of psychological egoism.', 'actually involves a confusion between ethical egoism and psychological egoism.', 'provides no evidence for or against psychological egoism.', 'contradicts the concept of ethical egoism.', 'is a misinterpretation of psychological egoism.', 'provides some evidence for psychological egoism.']\",\"question\":\"Feinberg claims that the story about Abraham Lincoln:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}