{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"761cc0be-df27-5e27-8180-fe4a7c97dc8a","task_key":"default--test--6965","task_revision_id":"1","upstream_id":"6965","short_description":"You buy 100 shares in XYZ Corporation on the Internet and your broker charges…","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['This will increase GDP by the total cost of the shares plus $29.95.', 'This will increase GDP by $29.95.', 'This will increase GDP by the cost of the shares minus $29.95.', 'This has no effect on GDP.', 'This will increase the investment component of GDP and therefore overall GDP.', 'This will increase GDP by the cost of the shares.', 'This will decrease GDP by the cost of the shares plus $29.95.', 'This will decrease the investment component of GDP and therefore overall GDP.', 'This will have a variable effect on GDP depending on the current market value of the shares.', 'This will decrease GDP by $29.95.']\",\"question\":\"You buy 100 shares in XYZ Corporation on the Internet and your broker charges you $29.95.\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}