{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"098ee93b-2e0d-578f-a218-067db3117d50","task_key":"all--test--098ee93b-2e0d-578f-a218-067db3117d50","task_revision_id":"1","upstream_id":"","short_description":"A 2008 survey of 500 households concluded that 82% of the population uses…","config":"all","split":"test","body":"{\"choices\":\"['Three percent of those surveyed refused to participate in the poll.', 'It would not be unexpected for 3% of the population to begin using coupons or stop using coupons.', 'Between 395 and 425 of the 500 households surveyed responded that they used grocery coupons.', 'It is likely that between 79% and 85% of the population use grocery coupons.']\",\"question\":\"A 2008 survey of 500 households concluded that 82% of the population uses grocery coupons. Which of the following best describes what is meant by the poll having a margin of error of 3%?\",\"subject\":\"high_school_statistics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}