{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0c809b6f-af7c-5025-a63e-38f8c9af81c9","task_key":"default--validation--39184","task_revision_id":"1","upstream_id":"39184","short_description":"HellaSwag validation 39184","config":"default","split":"validation","body":"{\"activity_label\":\"Shuffleboard\",\"ctx\":\"We see a girl walking slowly with a shuffleboard pole and pushing a puck. The lady gets to the blue line and pushes the puck. the puck\",\"endings\":\"['flips and sticks instead of sliding.', 'falls into the blue line and the lady looks at it.', 'moves back and forth over the gold line before hitting the ice hard.', 'falls through the floor and the lady walks back to pick it up.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}