{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d696f548-ce24-533e-b8e1-2418e4db8db4","task_key":"all--test--d696f548-ce24-533e-b8e1-2418e4db8db4","task_revision_id":"1","upstream_id":"","short_description":"If a malicious process is granted temporary administrator-level access to a…","config":"all","split":"test","body":"{\"choices\":\"['the threat is over as soon as the process is killed', 'the threat is over after the operating system is rebooted', 'the thread is over after rebooting the machine and replacing files that show a virus infection', 'the threat is often permanent until the entire disk is wiped clean']\",\"question\":\"If a malicious process is granted temporary administrator-level access to a system\",\"subject\":\"college_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}