{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4e648e39-8aca-51b4-b9da-4c53d82f42e8","task_key":"default--test--5659","task_revision_id":"1","upstream_id":"5659","short_description":"What is realism?","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['Realism is a set of approaches to Security Studies that focuses on states, and tries to determine which strategy a state should adopt to achieve security and other goals.', 'Realism is the broad term that can be applied to any traditional approach to Security Studies.', 'Realism is a set of approaches that encompasses any security threat that involves the military.', 'Realism is defined by the belief that the behaviour of states constructs the international arena and that this behaviour is established through the internal environment of the state.']\",\"question\":\"What is realism?\",\"src\":\"ori_mmlu-security_studies\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}