{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"23fe5893-ec87-59b7-ada8-5fcb51110350","task_key":"all--test--23fe5893-ec87-59b7-ada8-5fcb51110350","task_revision_id":"1","upstream_id":"","short_description":"If people expect the price of a particular product to increase in the near…","config":"all","split":"test","body":"{\"choices\":\"['this will not affect the demand for the product right now.', 'this will decrease the demand for the product.', 'this will not affect the demand for the product now or later.', 'this will increase the demand for the product.']\",\"question\":\"If people expect the price of a particular product to increase in the near future\",\"subject\":\"high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}