{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2693fd65-4357-58f2-a99b-4153e2b6f343","task_key":"default--test--2721","task_revision_id":"1","upstream_id":"2721","short_description":"Answering multiple-choice questions is often easier than answering fill-in or…","config":"default","split":"test","body":"{\"category\":\"psychology\",\"options\":\"['are definitional rather than conceptual', 'check memorization rather than critical thinking', 'encourage rote learning rather than comprehension', 'require less time to answer', 'enhance retention of information', 'provide more retrieval cues', 'are always easier to understand']\",\"question\":\"Answering multiple-choice questions is often easier than answering fill-in or completion questions, because multiple choice questions\",\"src\":\"ori_mmlu-high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}