{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3fd5f6e2-9644-58c4-84e0-6e3cbfd855a9","task_key":"all--test--3fd5f6e2-9644-58c4-84e0-6e3cbfd855a9","task_revision_id":"1","upstream_id":"","short_description":"According to Stevenson, empirical methods:","config":"all","split":"test","body":"{\"choices\":\"['are never useful in resolving ethical disagreement.', 'are sometimes useful in resolving ethical disagreement, but are not always sufficient.', 'are in principle always sufficient for resolving ethical disagreement.', 'provide the only responsible way to engage in moral philosophy.']\",\"question\":\"According to Stevenson, empirical methods:\",\"subject\":\"philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}