{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"466a0a4d-e149-507a-a2bc-cb0be6a9bed1","task_key":"all--test--466a0a4d-e149-507a-a2bc-cb0be6a9bed1","task_revision_id":"1","upstream_id":"","short_description":"A toy car and a toy truck collide. If the toy truck’s mass is double the toy…","config":"all","split":"test","body":"{\"choices\":\"['double the magnitude and in the same direction', 'double the magnitude and in the opposite direction', 'half the magnitude and in the same direction', 'half the magnitude and in the opposite direction']\",\"question\":\"A toy car and a toy truck collide. If the toy truck’s mass is double the toy car’s mass, then, compared to the acceleration of the truck, the acceleration of the car during the collision will be\",\"subject\":\"high_school_physics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}