{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"64fbefe4-1170-596c-a9e2-2fa34e37d1bc","task_key":"all--test--64fbefe4-1170-596c-a9e2-2fa34e37d1bc","task_revision_id":"1","upstream_id":"","short_description":"Singer claims that famine relief organizations direct our aid to distant…","config":"all","split":"test","body":"{\"choices\":\"['more effectively than we could get it to our close neighbors.', 'almost as effectively as we could get it to our close neighbors.', 'quite inefficiently, but they are still worth donating to.', 'very inefficiently, and are therefore not worth donating to.']\",\"question\":\"Singer claims that famine relief organizations direct our aid to distant refugees:\",\"subject\":\"philosophy\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}