{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"dbcf48a9-8974-5347-88e9-b099b34adfb8","task_key":"all--test--dbcf48a9-8974-5347-88e9-b099b34adfb8","task_revision_id":"1","upstream_id":"","short_description":"How are the members of the arbitral tribunal appointed?","config":"all","split":"test","body":"{\"choices\":\"['All the members of the arbitral tribunal are appointed by the parties', 'All the members of the arbitral tribunal are appointed by the parties', 'All the members of the arbitral tribunal are appointed by an impartial third party, such as the president of the ICJ', 'All the members of the arbitral tribunal are appointed by the parties from a restricted list of arbitrators']\",\"question\":\"How are the members of the arbitral tribunal appointed?\",\"subject\":\"international_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}