{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a0b0ca20-8a97-5bf2-8ac6-6d6590218c7a","task_key":"all--test--a0b0ca20-8a97-5bf2-8ac6-6d6590218c7a","task_revision_id":"1","upstream_id":"","short_description":"If firms that make a particular product expect its price will be lower in the…","config":"all","split":"test","body":"{\"choices\":\"['this will cause the supply of the product to increase right now.', 'this will cause the supply of the product to decrease right now.', 'this will have no effect on the amount of the product supplied right now.', 'this will have no effect on the supply of the product now or later.']\",\"question\":\"If firms that make a particular product expect its price will be lower in the future\",\"subject\":\"high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}