{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f030920e-02d0-55f4-8aa9-5b12ba1b494a","task_key":"default--test--6896","task_revision_id":"1","upstream_id":"6896","short_description":"Suppose yesterday the euro was trading on the foreign exchange market at $1.36…","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['Interest rates in the United States have decreased.', 'The U.S. dollar has appreciated.', 'The U.S. dollar has appreciated and the euro has depreciated.', 'The exchange rate has remained steady.', 'The U.S. dollar has stayed the same and the euro has depreciated.', 'The euro has appreciated.', 'Interest rates in Europe have increased.', 'The U.S. dollar has depreciated.', 'The euro has depreciated.', 'Interest rates in the United States have increased.']\",\"question\":\"Suppose yesterday the euro was trading on the foreign exchange market at $1.36 U.S. and today it is trading at $1.40 U.S. Which of the following is true?\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}