{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c2eebbfe-0644-5d47-9d99-4ae2157aab99","task_key":"default--validation--44043","task_revision_id":"1","upstream_id":"44043","short_description":"HellaSwag validation 44043","config":"default","split":"validation","body":"{\"activity_label\":\"Fixing the roof\",\"ctx\":\"A man is spraying the paste on the roof, a male is handing off the pieces, and a guy is adding the pieces to the pasted area on the roof. a person\",\"endings\":\"['is spraying the paste on the roof pieces.', 'is wearing a white apron and the color is on the screen.', 'sets the same tarp on the outside of the roofing and throws rakes out of the tarp onto the roof.', 'is laying out plaster on the roof.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}