{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"42d8bb57-5bdc-5ac6-b62f-d70d6e4da13d","task_key":"all--test--42d8bb57-5bdc-5ac6-b62f-d70d6e4da13d","task_revision_id":"1","upstream_id":"","short_description":"Bill is a successful ad executive for a computer company. He is 32 married and…","config":"all","split":"test","body":"{\"choices\":\"['female impersonator', 'drag queen', 'transsexual', 'transvestite']\",\"question\":\"Bill is a successful ad executive for a computer company. He is 32 married and the father of two children. When Bill's wife is not home he puts on her satin stockings garter belt bra skirt blouse and red stilettos. While dressing he becomes sexually excited and proceeds to masturbate. Which of the following descriptions best fits Bill?\",\"subject\":\"human_sexuality\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}