{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"221806be-e838-513d-a502-181399af1112","task_key":"default--test--1539","task_revision_id":"1","upstream_id":"1539","short_description":"A defendant charged with first-degree murder shall be furnished with a list…","config":"default","split":"test","body":"{\"category\":\"law\",\"options\":\"['only if the prosecutor agrees.', 'immediately after being charged.', 'upon request and showing of good cause.', 'upon court order.', 'under no circumstances.', 'only if the jurors give consent.', 'at the discretion of the judge.', 'upon request.', 'upon payment of a fee.', \\\"only if the defendant can prove it's necessary for their defense.\\\"]\",\"question\":\"A defendant charged with first-degree murder shall be furnished with a list containing names and addresses of all prospective jurors\",\"src\":\"ori_mmlu-professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}