{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d463d47e-86f3-53a6-8983-0b4401578d85","task_key":"all--test--d463d47e-86f3-53a6-8983-0b4401578d85","task_revision_id":"1","upstream_id":"","short_description":"Base Electric Co. has entered an agreement to buy its actual requirements of…","config":"all","split":"test","body":"{\"choices\":\"['Unenforceable because it is too indefinite as to quantity.', 'Unenforceable because it lacks mutuality of obligation.', 'Unenforceable because of lack of consideration.', 'Valid and enforceable.']\",\"question\":\"Base Electric Co. has entered an agreement to buy its actual requirements of copper wiring for six months from the Seymour Metal Wire Company and Seymour Metal has agreed to sell all the copper wiring Base will require for six months. The agreement between the two companies is\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}