{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0cfae349-8a62-5d50-9a13-c37215b80b57","task_key":"all--test--0cfae349-8a62-5d50-9a13-c37215b80b57","task_revision_id":"1","upstream_id":"","short_description":"The owner of a competitive firm making zero economic profit","config":"all","split":"test","body":"{\"choices\":\"['should consider shutting down because she could make more elsewhere', 'is making less than normal profits', 'is making exactly what she would make in her next best alternative job', 'will most likely make more profits in the long run']\",\"question\":\"The owner of a competitive firm making zero economic profit\",\"subject\":\"high_school_microeconomics\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}