{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4bae5d98-4194-5780-9c5c-f9acf5533557","task_key":"default--test--2008","task_revision_id":"1","upstream_id":"2008","short_description":"According to ethics codes published by the American and Canadian Psychological…","config":"default","split":"test","body":"{\"category\":\"psychology\",\"options\":\"['may attempt to resolve the problem informally', 'should post about the violation on social media to raise awareness', 'must report the violation to the APA Ethics Committee', 'should confront the colleague directly in a public setting', 'must report the incident to the state/provincial licensing board', 'may choose to ignore the violation', 'must report the violation to the police', 'should document the violation but take no further action', 'must immediately end all professional relationships with the colleague', \\\"must report the violation to the colleague's supervisor\\\"]\",\"question\":\"According to ethics codes published by the American and Canadian Psychological Associations, if a psychologist knows of an ethical violation of a minor nature by a colleague, the psychologist\",\"src\":\"ori_mmlu-professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}