{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"83b811d5-6315-5e5f-b1f2-f8a3309170ac","task_key":"default--test--5404","task_revision_id":"1","upstream_id":"5404","short_description":"Which of these is not one of superhero Captain Marvel's abilities?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['the strength of Hercules', 'the endurance of Atlas', 'the insight of Athena', 'the speed of Hermes', 'the courage of Achilles', 'the healing of Asclepius', 'the agility of Artemis', 'the wisdom of Solomon', 'the power of Apollo', 'the stealth of Hades']\",\"question\":\"Which of these is not one of superhero Captain Marvel's abilities?\",\"src\":\"ori_mmlu-miscellaneous\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}