{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1bd121bc-673c-5161-b78a-deea553c70b0","task_key":"default--test--11225","task_revision_id":"1","upstream_id":"11225","short_description":"Identify the antecedent of the following conditional proposition: The university…","config":"default","split":"test","body":"{\"category\":\"philosophy\",\"options\":\"['The board of trustees does not recommend it.', 'The governor approves of it.', 'The board of trustees recommends it.', 'The governor disapproves of it and the board of trustees does not recommend it.', 'The governor approves of it and the board of trustees recommends it.', 'The university does not raise tuition.', 'Both the governor and the board of trustees do not approve of it.', 'The university raises tuition.', 'The governor disapproves of it.', 'The governor approves of it or the board of trustees recommends it.']\",\"question\":\"Identify the antecedent of the following conditional proposition: The university raises tuition only if both the governor approves of it and the board of trustees recommends it.\",\"src\":\"ori_mmlu-formal_logic\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}