{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b391915a-41c8-5db5-a709-2ead9ce6e4c2","task_key":"all--test--b391915a-41c8-5db5-a709-2ead9ce6e4c2","task_revision_id":"1","upstream_id":"","short_description":"\"A \"\"cover story\"\" that causes research participants to think that the purpose…","config":"all","split":"test","body":"{\"choices\":\"['Hawthorne effect', 'Rosenthal effect', 'demand characteristics', 'differential attrition']\",\"question\":\"\\\"A \\\"\\\"cover story\\\"\\\" that causes research participants to think that the purpose of the study they are participating in is something other than what it really is would be most helpful for controlling which of the following?\\\"\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}