{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3f04e7bf-ebee-598b-a95a-12635c28358b","task_key":"default--validation--14401","task_revision_id":"1","upstream_id":"14401","short_description":"HellaSwag validation 14401","config":"default","split":"validation","body":"{\"activity_label\":\"Scuba diving\",\"ctx\":\"An intro leads into a man in women sitting beside a pool and talking. the people\",\"endings\":\"['hold up shots and begin shooting at a pinata.', 'then jump in and begin swimming in the pool while one of the women jokes with the camera man.', 'are then shown underwater wearing scuba gear and teaching one another hand signals while the man speaks to the camera above the surface.', 'then play a game while the camera shows them all completing the turn at the end.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}