{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"17e15882-fcbe-5c79-a263-252ead31f524","task_key":"default--test--10939","task_revision_id":"1","upstream_id":"10939","short_description":"According to Hume, reason:","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['should be used to direct our passions.', 'should operate completely independently of our passions.', 'should be completely suppressed by our passions.', 'should be ignored when it comes to our passions.', 'should be used to validate our passions.', 'should be used to contradict our passions.', 'should serve our passions.', 'should be used to intensify our passions.', 'should be used to rationalize our passions.', 'should be used to extinguish our passions.']\",\"question\":\"According to Hume, reason:\",\"src\":\"ori_mmlu-philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}