{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bf68c24b-e623-555d-9f1f-2ea85068d0a0","task_key":"all--test--bf68c24b-e623-555d-9f1f-2ea85068d0a0","task_revision_id":"1","upstream_id":"","short_description":"Mr. Kohlberg owns a flower shop. At the beginning of the day, he had 152 roses.…","config":"all","split":"test","body":"{\"choices\":\"['7', '12', '48', '56']\",\"question\":\"Mr. Kohlberg owns a flower shop. At the beginning of the day, he had 152 roses. Mr. Kohlberg sold 96 of the roses and then wanted to separate the rest of the roses equally among 8 vases. What will be the total number of roses in each vase?\",\"subject\":\"elementary_mathematics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}