{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9e329272-d935-508c-8753-82e0e3522fad","task_key":"default--test--11397","task_revision_id":"1","upstream_id":"11397","short_description":"Let A =\\vert22\\vert,B =\\vert31\\vert. Find AB. \\vert12\\vert\\vert12\\vert…","config":"default","split":"test","body":"{\"category\":\"engineering\",\"options\":\"['AB = |76| |75| |168|', 'AB = |96| |65| |158|', 'AB = |76| |45| |138|', 'AB = |96| |55| |138|', 'AB = |86| |45| |138|', 'AB = |76| |55| |148|', 'AB = |86| |55| |148|', 'AB = |96| |75| |168|', 'AB = |106| |75| |168|', 'AB = |106| |65| |158|']\",\"question\":\"Let A =\\\\vert22\\\\vert,B =\\\\vert31\\\\vert. Find AB. \\\\vert12\\\\vert\\\\vert12\\\\vert \\\\vert42\\\\vert\",\"src\":\"stemez-ElectricCircuits\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}