{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"53aaa4b0-6b76-562d-aa41-d7cb02ac26a7","task_key":"all--test--53aaa4b0-6b76-562d-aa41-d7cb02ac26a7","task_revision_id":"1","upstream_id":"","short_description":"A university sent out a survey to a random sample of 120 recent graduates to see…","config":"all","split":"test","body":"{\"choices\":\"['Use the 88 who did respond, using 88 as the sample size in the analysis.', 'Use the 88 who did respond, using 120 as the sample size in the analysis.', 'Attempt to contact the 32 who did not respond and get the information from them.', 'Select 32 more graduates at random and use them to replace the nonresponders.']\",\"question\":\"A university sent out a survey to a random sample of 120 recent graduates to see how those graduates feel about the education they received. Thirty-two of the graduates did not respond to the survey. Which of these would be the best way to proceed?\",\"subject\":\"high_school_statistics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}