{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"df4d97ec-bd0b-5055-81c3-32fb94878a66","task_key":"default--test--7426","task_revision_id":"1","upstream_id":"7426","short_description":"When we observe an unplanned decrease in inventories we can expect","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['output to begin to rise.', 'output to begin to fall.', 'saving to begin to rise.', 'production to slow down.', 'demand to decrease.', 'saving to begin to fall.', 'the economy to enter a recession.', 'inflation to increase.', 'prices to begin to fall.']\",\"question\":\"When we observe an unplanned decrease in inventories we can expect\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}