{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f1824beb-f963-5143-93d4-bef22555455f","task_key":"default--test--7221","task_revision_id":"1","upstream_id":"7221","short_description":"If people expect the price of a particular product to increase in the near…","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['this will increase the supply of the product.', 'this will cause the producer to decrease the supply of the product.', 'this will not affect the demand for the product now or later.', 'this will decrease the supply of the product.', 'this will increase the demand for the product.', 'this will not affect the demand for the product right now.', 'this will make the product obsolete.', 'this will decrease the demand for the product.', 'this will not affect the supply of the product.', 'this will cause the producer to increase the supply of the product.']\",\"question\":\"If people expect the price of a particular product to increase in the near future\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}