{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3d72f113-c6da-5f1d-8e54-9c64ce1e7141","task_key":"default--test--7286","task_revision_id":"1","upstream_id":"7286","short_description":"Required reserves","config":"default","split":"test","body":"{\"category\":\"economics\",\"options\":\"['can be used by banks to make loans or buy investments.', 'must be held at the FED exclusively.', \\\"must be kept in a bank's vault.\\\", 'are optional for large banks but compulsory for smaller ones.', 'can be invested in government securities.', 'must be used to pay off bank debts.', 'can be used to cover operational expenses of the bank.', \\\"can be held in a bank's vault or its account at the FED.\\\", 'must be used to make loans.', 'are not required to be held by banks.']\",\"question\":\"Required reserves\",\"src\":\"ori_mmlu-high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}