{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"841420aa-2e9c-551a-9a33-c18c23ce6c8e","task_key":"default--validation--31285","task_revision_id":"1","upstream_id":"31285","short_description":"HellaSwag validation 31285","config":"default","split":"validation","body":"{\"activity_label\":\"Having an ice cream\",\"ctx\":\"A woman is sitting with a baby in her arms. she\",\"endings\":\"['grabs a bottle, pours it into the pan, and begins frying eggs.', 'begins to hip hula hoop and dance.', 'takes licks of an ice cream cone, and the baby laughs hysterically each time.', 'starts feeding the baby vegetables from a bottle.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}