{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7e3a7f2b-9896-56e5-b7c5-ce8baa1998e6","task_key":"default--test--5940","task_revision_id":"1","upstream_id":"5940","short_description":"An example of a categorical grant would be","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['money given by the federal government to states to fund military defense', 'money loaned by the federal government to a state to prevent it from bankruptcy', 'money given by the federal government to states to fund Head Start early education programs', 'money loaned by the federal government to states for infrastructure projects', 'money given by the federal government to states without conditions', 'money loaned by the federal government to a private company to prevent it from failing', 'money given by the federal government to cities with no specific use required']\",\"question\":\"An example of a categorical grant would be\",\"src\":\"ori_mmlu-high_school_government_and_politics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}