{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"97a82489-a7c0-5794-9489-2de62d9e643f","task_key":"default--validation--38246","task_revision_id":"1","upstream_id":"38246","short_description":"HellaSwag validation 38246","config":"default","split":"validation","body":"{\"activity_label\":\"Doing crunches\",\"ctx\":\"A female is shown straddling a punching bag and doing sit-ups with it. she\",\"endings\":\"['continues kneeling down and doing sit ups while she stretches.', 'then jumps up and spins around multiple times.', 'does push ups with the punching bag sitting on the floor.', 'does quite a few reps before the video cuts out.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}