{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ac1a1443-80bd-5e76-93f2-92b12d3d486b","task_key":"all--test--ac1a1443-80bd-5e76-93f2-92b12d3d486b","task_revision_id":"1","upstream_id":"","short_description":"An increase in the price of forklifts imported into the United States from…","config":"all","split":"test","body":"{\"choices\":\"['increase the consumer price index and the GDP deflator.', 'increase the consumer price index but not the GDP deflator.', 'increase the GDP deflator but not the consumer price index.', 'have no effect on the consumer price index or the GDP deflator.']\",\"question\":\"An increase in the price of forklifts imported into the United States from Belgium will\",\"subject\":\"high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}