{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f2ccc098-dd33-5a5a-ab5c-2d5775ea4ed0","task_key":"default--validation--38188","task_revision_id":"1","upstream_id":"38188","short_description":"HellaSwag validation 38188","config":"default","split":"validation","body":"{\"activity_label\":\"Having an ice cream\",\"ctx\":\"The ingredients are added to a tub and scoops of ice cream are taken from a tub after freezing. A bowl is filled with ice cream. the man\",\"endings\":\"['leans out of the tub and picks up a cake and puts it inside.', 'pours into two buckets and dumps them in the ice cream.', 'enjoys the ice cream.', 'prepares the ice cream and takes his turn cooking the delicious food.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}