{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a207ebe3-e58c-5fc7-b937-6cf59448da43","task_key":"default--test--11018","task_revision_id":"1","upstream_id":"11018","short_description":"Mill defines “utility” as:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['advantage or benefit to the user.', 'usefulness for some craft.', 'the ability to satisfy wants and needs.', 'usefulness to an individual.', 'it promotes the interests of all and does not harm anyone.', 'it promotes self-interest.', 'the state of being useful, profitable, or beneficial.', 'pleasure and the absence of pain.', 'usefulness to society.', 'it benefits the majority and harms the minority.']\",\"question\":\"Mill defines “utility” as:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}