{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"022d2a09-a21e-5fb4-9fa8-edcbbdbfbdc1","task_key":"all--test--022d2a09-a21e-5fb4-9fa8-edcbbdbfbdc1","task_revision_id":"1","upstream_id":"","short_description":"According to Singer, which of the following historical defenders of the…","config":"all","split":"test","body":"{\"choices\":\"['Immanuel Kant', 'Aristotle', 'Jeremy Bentham', 'John Stuart Mill']\",\"question\":\" According to Singer, which of the following historical defenders of the principle of equal consideration of interests was one of the few to recognize that the principle applies to members of other species as well as to our own?\",\"subject\":\"moral_disputes\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}