{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d10e120d-f28b-5a55-a36f-67161e3d3674","task_key":"default--test--11737","task_revision_id":"1","upstream_id":"11737","short_description":"In decimal number system what is MSD","config":"default","split":"test","body":"{\"category\":\"engineering\",\"options\":\"['First two digits from right to left', 'Median of all digits', 'First digit from right to left', 'First digit from left to right', 'Mode of all digits', 'Last digit from left to right', 'Mean of all digits', 'Middle digit', 'First two digits from left to right']\",\"question\":\"In decimal number system what is MSD\",\"src\":\"ori_mmlu-electrical_engineering\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}