{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"da6c35b5-79c2-5ba1-80dc-80559283794d","task_key":"default--validation--26033","task_revision_id":"1","upstream_id":"26033","short_description":"HellaSwag validation 26033","config":"default","split":"validation","body":"{\"activity_label\":\"Decorating the Christmas tree\",\"ctx\":\"Kids and adults are running around a room. They are putting decorations on a christmas tree. they\",\"endings\":\"['take picture of the tree.', 'are enjoying living in the city and exploring the art and water features.', 'hang lights and then the ornaments.', 'go up the stairs and start decorating the tree.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}