{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"edf0f5fa-886a-561c-bcc7-3795c63e1f28","task_key":"default--validation--25956","task_revision_id":"1","upstream_id":"25956","short_description":"HellaSwag validation 25956","config":"default","split":"validation","body":"{\"activity_label\":\"Calf roping\",\"ctx\":\"A person is seen riding in on a horse chasing a calf in the middle of a large crowd. Another man is seen chasing a calf on a horse and roping a calf running away. several more men\",\"endings\":\"['are seen through the break in the crowd riding horses and chasing back the calf.', 'are seen hurt and running away while the camera pans all around them.', 'bring the calf back to the crowd and tie off the calf.', 'take their turns chasing after the calfs.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}