{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fa0e2ab0-6ab7-5cac-a210-4a3cc319ca5f","task_key":"default--test--11250","task_revision_id":"1","upstream_id":"11250","short_description":"Whether someone is hypocritical regarding her claims is...","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['Irrelevant to the truth of the claims', 'Only relevant if the person is a public figure', 'Only valid if the person is conscious of their hypocrisy', 'A sign that the person is untrustworthy', \\\"Direct evidence of the person's lying tendencies\\\", 'Evidence that the claims are false', 'Relevant only in philosophical discussions', 'A proof that the person lacks integrity', 'Irrelevant to her character', 'Relevant only in court']\",\"question\":\"Whether someone is hypocritical regarding her claims is...\",\"src\":\"ori_mmlu-logical_fallacies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}