{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"5de46883-88a2-5a7b-96b1-975121a998fd","task_key":"all--test--5de46883-88a2-5a7b-96b1-975121a998fd","task_revision_id":"1","upstream_id":"","short_description":"You are director of communications for a local non-profit chapter organization…","config":"all","split":"test","body":"{\"choices\":\"['Use your relationships with local donors to gain coverage.', 'Localize the news releases for audiences in your geography.', \\\"Pitch the news releases as written, but include a quote from your chapter's executive director.\\\", 'Remind editors that your organization is a national, prominent non-profit to which their publisher donates.']\",\"question\":\"You are director of communications for a local non-profit chapter organization that is an affiliate of a large national non-profit organization. You are frequently asked to secure media coverage using news releases provided by national headquarters located in another state. Which action would most improve your chances of success with these materials?\",\"subject\":\"public_relations\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}