{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ebec4082-4b61-56ad-940d-701bbf50a13f","task_key":"all--test--ebec4082-4b61-56ad-940d-701bbf50a13f","task_revision_id":"1","upstream_id":"","short_description":"International development may be tied to the security of the state, why?","config":"all","split":"test","body":"{\"choices\":\"['Threats to the state centralize political and economic power, threatening state development', 'Too many casualties drain the productive power of the state', 'Threats to the state encourage socialism', 'The two are correlated, but there is no causal effect']\",\"question\":\"International development may be tied to the security of the state, why?\",\"subject\":\"us_foreign_policy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}