{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1410f54d-f82f-504e-afc2-cbe289d2fd98","task_key":"all--test--1410f54d-f82f-504e-afc2-cbe289d2fd98","task_revision_id":"1","upstream_id":"","short_description":"You've just finished training a decision tree for spam classification, and it is…","config":"all","split":"test","body":"{\"choices\":\"['Your decision trees are too shallow.', 'You need to increase the learning rate.', 'You are overfitting.', 'None of the above.']\",\"question\":\"You've just finished training a decision tree for spam classification, and it is getting abnormally bad performance on both your training and test sets. You know that your implementation has no bugs, so what could be causing the problem?\",\"subject\":\"machine_learning\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}