{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c5775f7d-8305-547c-9ec5-1ce9817a030d","task_key":"default--test--7168","task_revision_id":"1","upstream_id":"7168","short_description":"If firms that make a particular product expect its price will be lower in the…","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['this will cause the supply of the product to decrease right now.', 'this will have no effect on the amount of the product supplied right now.', 'this will have no effect on the supply of the product now or later.', 'this will cause the supply of the product to increase right now.', 'this will cause the supply of the product to decrease in the future.', 'this will cause the demand for the product to decrease right now.', 'this will cause the demand for the product to decrease in the future.', 'this will cause the demand for the product to increase in the future.', 'this will cause the supply of the product to increase in the future.', 'this will cause the demand for the product to increase right now.']\",\"question\":\"If firms that make a particular product expect its price will be lower in the future\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}