{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"40bcae71-c739-592a-a57c-ab519f98e4a9","task_key":"all--test--40bcae71-c739-592a-a57c-ab519f98e4a9","task_revision_id":"1","upstream_id":"","short_description":"When the value of the U.S. dollar appreciates relative to other currencies which…","config":"all","split":"test","body":"{\"choices\":\"['Imports into the United States will decrease.', 'Exports from the United States will increase.', 'U.S. residents will take more vacations in foreign countries.', 'More foreign visitors will travel to the United States.']\",\"question\":\"When the value of the U.S. dollar appreciates relative to other currencies which of the following is the most likely to occur?\",\"subject\":\"high_school_macroeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}