{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e04feefc-e7c9-5993-8c82-e737265cedf8","task_key":"default--validation--3534","task_revision_id":"1","upstream_id":"3534","short_description":"HellaSwag validation 3534","config":"default","split":"validation","body":"{\"activity_label\":\"Cleaning shoes\",\"ctx\":\"A man is seen speaking to the camera and begins polishing a black shoe with a brush. he\",\"endings\":\"['then holds up the shoe and starts waxing it with a shoe shaver.', 'then shows the shoe again and puts it back on.', 'then uses scraper to polish the shoe while still speaking and cutting with sharp edges.', 'points around the shoe and flips the shoe around with his hands.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}