{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8147d8c5-d58a-506f-afc3-4824e7a724bd","task_key":"default--test--6398","task_revision_id":"1","upstream_id":"6398","short_description":"Under which circumstances would you not use a catheter valve?","config":"default","split":"test","body":"{\"category\":\"health\",\"options\":\"['If the patient chooses it.', 'If the patient is bedridden.', 'If urine does not need to be measured.', 'If the patient has bladder sensation.', 'If the patient suffers from urinary incontinence.', 'If the patient is post-operative.', 'If the patient is in the late stages of pregnancy.', 'If the patient has a history of bladder spasms.']\",\"question\":\"Under which circumstances would you not use a catheter valve?\",\"src\":\"ori_mmlu-clinical_knowledge\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}