{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"cb45c81b-6790-5a99-bdc6-95cb1066270b","task_key":"all--test--cb45c81b-6790-5a99-bdc6-95cb1066270b","task_revision_id":"1","upstream_id":"","short_description":"A man is on trial for securities fraud. He takes the stand to testify in his own…","config":"all","split":"test","body":"{\"choices\":\"['The defendant may introduce evidence to show truthfulness.', 'The defendant may introduce evidence to show honesty.', 'The defendant may not introduce evidence to show honesty because character evidence is inadmissible.', 'The defendant may not introduce evidence of truthfulness unless it is inquired into on cross- examination.']\",\"question\":\"A man is on trial for securities fraud. He takes the stand to testify in his own behalf. On direct examination, the man's attorney proffers evidence of his good character. Upon objection by the prosecution, which of the following statements is correct?\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}