{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4e31ca88-2b7b-57f4-852c-a8600c380b8d","task_key":"all--test--4e31ca88-2b7b-57f4-852c-a8600c380b8d","task_revision_id":"1","upstream_id":"","short_description":"Which of the following samples would be considered most representative of male…","config":"all","split":"test","body":"{\"choices\":\"['A group of thirty fraternity brothers from Penn State', 'A random sample taken between classes in the business wing of various universities', 'Sixty male members of each class from Princeton, Yale, Harvard, Dartmouth, and Columbia', 'Twenty male members of each class from a cross-section of colleges and universities']\",\"question\":\"Which of the following samples would be considered most representative of male college students?\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}