{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c359f181-0ea8-59a4-9bd3-7bd7a880d41a","task_key":"default--test--11042","task_revision_id":"1","upstream_id":"11042","short_description":"Richardson-Self argues that sexist speech","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['Can never be considered hate speech regardless of context', 'Is never hate speech because it is a form of free expression.', 'is always hate speech, because it does not involves violence.', 'is not always hate speech, because it does not involve violence', 'is always hate speech, because it involves violence.', 'Is not hate speech because it is protected under freedom of speech', 'Is always hate speech because it promotes gender inequality', 'Is only hate speech when it directly incites violence', 'Can be considered hate speech when it promotes discrimination', 'I s not always hate speech, because it always involves violence.']\",\"question\":\" Richardson-Self argues that sexist speech\",\"src\":\"ori_mmlu-moral_disputes\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}