{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2dea7bf6-0d87-5523-b77b-93a10a99ac88","task_key":"default--validation--23255","task_revision_id":"1","upstream_id":"23255","short_description":"HellaSwag validation 23255","config":"default","split":"validation","body":"{\"activity_label\":\"Playing saxophone\",\"ctx\":\"A large group of people are seen wandering around a city square with a young child in the middle playing an instrument. the boy\",\"endings\":\"['continues to fiddle with the instrument while walking to and fro close to the camera.', 'plays a saxophone while people walk past him and put money down.', 'continues to play while people walk in and out of frame.', 'continues to play throughout the whole time leads into him capturing pictures of himself and another.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}