{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d0548925-3d02-5d1a-a824-deee42099373","task_key":"default--test--11021","task_revision_id":"1","upstream_id":"11021","short_description":"Cicero argues that we will never go morally astray if we:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['follow religious edicts.', 'follow historical precedents.', 'follow the teachings of philosophers.', 'follow Nature as our guide.', 'follow our conscience.', 'follow the laws.', 'follow our personal desires.', 'follow our instincts.', 'follow societal norms.', 'follow our friends and family.']\",\"question\":\"Cicero argues that we will never go morally astray if we:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}