{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a902aaab-78cb-5b10-a06d-3e5b55027644","task_key":"all--test--a902aaab-78cb-5b10-a06d-3e5b55027644","task_revision_id":"1","upstream_id":"","short_description":"Use the following key to translate the given formula of PL to natural, English…","config":"all","split":"test","body":"{\"choices\":\"[\\\"If Emily doesn't stop working then Russell makes dinner.\\\", 'Emily stops working unless Russell makes dinner.', \\\"Emily stops working unless Russell doesn't make dinner.\\\", \\\"Emily doesn't stop working unless Russell makes dinner.\\\"]\",\"question\":\" Use the following key to translate the given formula of PL to natural, English sentences.\\nA: Marina reads a Percy Jackson book.\\nB: Izzy plays Minecraft.\\nC: Emily stops working.\\nD: Russell makes dinner.\\nE: Ashleigh stops by.\\n~C ∨ D\",\"subject\":\"formal_logic\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}