{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6d9098bf-959a-545d-9b7f-b6e919e39f62","task_key":"default--validation--25559","task_revision_id":"1","upstream_id":"25559","short_description":"HellaSwag validation 25559","config":"default","split":"validation","body":"{\"activity_label\":\"Ice fishing\",\"ctx\":\"A man is seen sitting on a bucket on the ice fishing through a hole in the ice. He catches a fish and then puts it back in the water under the ice. he\",\"endings\":\"['catches and moves back to the life buoy.', 'chases after into the hole again and pulls it out while pulling it up with his fishing pole.', 'stands up on the other side of the hole still fishing and ends by walking off.', 'then catches a few more fish afterwards.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}