{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"72125acb-54b7-5394-bdfd-8eb3f1c5d84a","task_key":"default--validation--32559","task_revision_id":"1","upstream_id":"32559","short_description":"HellaSwag validation 32559","config":"default","split":"validation","body":"{\"activity_label\":\"Roof shingle removal\",\"ctx\":\"We see blue letters on a black screen and see a roof. Then we switch to more letters and roof has been sprayed with a solution. we\",\"endings\":\"['see a large bucket of dirt and a dog takes the bucket and sets it aside.', 'see the panning attachment on the roof.', 'see nos and the design is perfect.', 'are informed the roof has been cleaned and shown shots of the clean roof.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}