{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"954e2d2a-75bf-5dbc-92b5-e2ae4cfbd616","task_key":"default--validation--18164","task_revision_id":"1","upstream_id":"18164","short_description":"HellaSwag validation 18164","config":"default","split":"validation","body":"{\"activity_label\":\"Cleaning windows\",\"ctx\":\"A man in a black shirt stands by a glass door. He then begins to rub the frame with a blue rag. as he\",\"endings\":\"['applies the rag, he stops and opens the door, continuing to rub the mirror.', 'does this he explains his methods.', 'continues, he continuously blows the window glass from side to side.', 'rubs, he takes his glasses off.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}