{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"71d556ee-3567-51e8-a94a-a264e70151b7","task_key":"default--validation--13776","task_revision_id":"1","upstream_id":"13776","short_description":"HellaSwag validation 13776","config":"default","split":"validation","body":"{\"activity_label\":\"Playing harmonica\",\"ctx\":\"A man is seen sitting in front of a camera holding a harmonica in his hands. the man\",\"endings\":\"['then begins playing the instrument while moving his hands up and down over the strings, looking up at the camera and continuing playing.', 'then begins playing the harmonica throughout the entire song and ends by adding a bow.', 'then begins playing several songs while pausing in between.', 'begins playing the harmonica back and fourth.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}