{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8b3d9bf7-df1b-556b-aa68-87438427d257","task_key":"default--test--6285","task_revision_id":"1","upstream_id":"6285","short_description":"What must you ensure before a speaking valve is used?","config":"default","split":"test","body":"{\"category\":\"health\",\"options\":\"['The patient has no difficulty in breathing.', 'It is 24 hours after the tracheostomy tube was first inserted.', 'The patient is able to speak clearly.', \\\"The patient's vital signs are stable.\\\", 'That the patient is not on a ventilator.', 'It is seven days after the tracheostomy tube was first inserted.', 'That the patient is able to swallow properly.', 'The cuff is inflated.', 'The cuff is deflated.', 'The cuff is partially deflated.']\",\"question\":\"What must you ensure before a speaking valve is used?\",\"src\":\"ori_mmlu-clinical_knowledge\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}