{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f815e63f-b5d2-572d-bfc9-d546ee0a7f5c","task_key":"default--test--1494","task_revision_id":"1","upstream_id":"1494","short_description":"Do treaties bind third States, ie non-State parties?","config":"default","split":"test","body":"{\"category\":\"law\",\"options\":\"['Treaties do no create obligations or rights for third States without their consent', 'Treaties create both obligations and rights for third States', 'Treaties do not create any rights for third States, even when the latter consent.', 'Treaties may create only obligations for third States', 'Treaties create obligations for third States without their consent', 'Treaties may create only rights for third States', 'Treaties do not create any obligations or rights for third States, even when the latter consent.', 'Treaties create rights for third States without their consent']\",\"question\":\"Do treaties bind third States, ie non-State parties?\",\"src\":\"ori_mmlu-international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}