{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"81243493-b321-51bc-be04-af2cf7956607","task_key":"default--test--5069","task_revision_id":"1","upstream_id":"5069","short_description":"The majority of Supreme Court justices","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['have no prior judicial experience', 'are appointed by presidents of the same political party', 'have previously served as Vice President', 'are appointed by the Senate', 'are nominated by the Speaker of the House', 'have a background in criminal law', 'stay on the court for only a decade', 'are required to have a degree in constitutional law', 'are required to retire at age 70', 'switch party affiliation during their term']\",\"question\":\"The majority of Supreme Court justices\",\"src\":\"ori_mmlu-high_school_government_and_politics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}