{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu","formal_name":"MMLU","introduction":"MMLU asks four-way multiple-choice questions across 57 subjects, from elementary material to professional-level examinations. Since 2020 it has been the most widely cited single reference point for general capability.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/hendrycks/test","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c47eaa38-2e16-5819-9d33-09ae03ec9af9","task_key":"all--test--c47eaa38-2e16-5819-9d33-09ae03ec9af9","task_revision_id":"1","upstream_id":"","short_description":"A widget manufacturer and a widget retailer signed the following agreement:\"The…","config":"all","split":"test","body":"{\"choices\":\"['a divisible contract.', 'an installment contract.', '10 separate contracts.', 'a requirements-output contract.']\",\"question\":\"A widget manufacturer and a widget retailer signed the following agreement:\\\"The manufacturer promises to sell and the retailer promises to buy 10,000 widgets at $50 per 100. One thousand widgets are to be delivered by the manufacturer on the first day of each of the next 10 months. EACH MONTHLY DELIVERY ISA SEPARATE CONTRACT. Payment to be made within 10 days of receipt of each shipment. \\\"The agreement between the manufacturer and the retailer may best be interpreted as\",\"subject\":\"professional_law\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/hendrycks/test","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}