{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4d2481e0-8607-55f7-b6c5-e9797b4b899d","task_key":"default--test--2030","task_revision_id":"1","upstream_id":"2030","short_description":"When the class listened to a list of words, half the group was directed to…","config":"default","split":"test","body":"{\"category\":\"psychology\",\"options\":\"['auditory processing', 'surface processing', 'phonetic features', 'shallow processing', 'structural processing', 'semantic features', 'echoic features', 'acoustic features', 'visual processing', 'deep processing']\",\"question\":\"When the class listened to a list of words, half the group was directed to listen for sounds while the other half was asked to gauge the emotional impact of the words. The group that gauged the emotional impact remembered many more words. This is evidence that better retention results with attention to\",\"src\":\"ori_mmlu-high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}