{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a2f1733b-be01-5524-99b0-af151f008991","task_key":"default--test--5694","task_revision_id":"1","upstream_id":"5694","short_description":"Which of the following best describes the situation after the price is raised on…","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['Consumers will start to demand more of the product.', 'The price of complementary goods will increase.', 'Total revenue will increase.', 'The product will no longer be in demand.', 'Total revenue will not change.', 'The quantity demanded will increase.', 'The quantity demanded will drastically decrease.', 'The quantity demanded will not change.', 'Total revenue will decrease.', 'The price of substitute goods will decrease.']\",\"question\":\"Which of the following best describes the situation after the price is raised on a product that has an inelastic demand?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}