{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fe81f583-569c-5a79-bafa-9a96c3fd059c","task_key":"all--test--fe81f583-569c-5a79-bafa-9a96c3fd059c","task_revision_id":"1","upstream_id":"","short_description":"Jacob cries uncontrollably every time his mother takes him down the candy isle…","config":"all","split":"test","body":"{\"choices\":\"['punished', 'associated', 'reinforced', 'extinguished']\",\"question\":\"Jacob cries uncontrollably every time his mother takes him down the candy isle in the supermarket and refuses to let him buy anything. Eventually, Jacob's mother gives in and lets him choose one candy item to buy if he stops crying. Jacob's crying behavior is __________________ by his mother.\",\"subject\":\"high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}