{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"50041bf7-e40c-5d97-a56b-b09b5f352e6c","task_key":"all--test--50041bf7-e40c-5d97-a56b-b09b5f352e6c","task_revision_id":"1","upstream_id":"","short_description":"the study‘s prospective value.","config":"all","split":"test","body":"{\"choices\":\"['do not have to be told the true purpose as long as the use of deception is justified by', 'do not have to be told the true purpose unless not to do so is likely to cause them substantial psychological harm.', 'should be debriefed about the true purpose as soon as possible after their participation.', 'must be debriefed about the true purpose immediately after their participation.']\",\"question\":\"the study‘s prospective value.\",\"subject\":\"professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}