{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7c3f5d6a-1329-5ac7-bad3-5177d4bfa994","task_key":"all--test--7c3f5d6a-1329-5ac7-bad3-5177d4bfa994","task_revision_id":"1","upstream_id":"","short_description":"A 22-year-old woman comes to the office because of urticaria. This is her first…","config":"all","split":"test","body":"{\"choices\":\"['Determine the serum IgE concentration', 'Determine the total eosinophil count', 'Refer her to an allergist', 'Treat the symptoms']\",\"question\":\"A 22-year-old woman comes to the office because of urticaria. This is her first episode of urticaria and it has occurred and then resolved several times in the past week. The history and physical examination disclose no abnormalities. Which of the following is the most appropriate course of action?\",\"subject\":\"professional_medicine\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}