{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2b7e5aa7-6104-515e-912b-365cb5d1bdab","task_key":"all--test--2b7e5aa7-6104-515e-912b-365cb5d1bdab","task_revision_id":"1","upstream_id":"","short_description":"In response to the worry that the freedom of association could justify…","config":"all","split":"test","body":"{\"choices\":\"['such a policy would wrongly disrespect existing citizens who belong to that race.', 'this is an implication of the freedom of association that we must accept.', 'this is an unavoidable implication of all philosophical positions on immigration policy.', 'none of the above']\",\"question\":\" In response to the worry that the freedom of association could justify immigration policies that exclude entry to members of a given race, Wellman claims that\",\"subject\":\"moral_disputes\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}