{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"954677c9-69f2-550e-89ee-fb4ffd1fab68","task_key":"default--test--6824","task_revision_id":"1","upstream_id":"6824","short_description":"How many calories should a woman eat each day during pregnancy?","config":"default","split":"test","body":"{\"category\":\"health\",\"options\":\"['No change in the first two trimesters, then an additional 500 kCal in the third trimester', 'No change from pre-pregnant levels', 'A decrease of 200 kCal throughout pregnancy', 'An additional 300 kCal in the second and third trimesters only', 'Less than 200 kCal', 'An additional 200 kCal in the second trimester only', 'An additional 200 kCal throughout pregnancy', 'An additional 200 kCal in the third trimester only', 'An additional 500 kCal throughout pregnancy', 'An additional 200 kCal in the first trimester only']\",\"question\":\"How many calories should a woman eat each day during pregnancy?\\n\",\"src\":\"ori_mmlu-nutrition\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}