{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a36dc908-d5b6-5e72-ac49-a07d61afe1b2","task_key":"all--test--a36dc908-d5b6-5e72-ac49-a07d61afe1b2","task_revision_id":"1","upstream_id":"","short_description":"\"Is there anything you would be willing to die for?\" is a philosophical question…","config":"all","split":"test","body":"{\"choices\":\"['it does not have any right or wrong answer because it is a meaningless question.', 'it is a meaningless question because everyone could have a different answer to it.', 'it forces us to articulate and justify our beliefs about what we know and ought to do.', \\\"it is more concerned with one's religious beliefs than with factual claims about the world.\\\"]\",\"question\":\"\\\"Is there anything you would be willing to die for?\\\" is a philosophical question insofar as:\",\"subject\":\"philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}