{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"44c0aa38-1067-595d-8d9d-fb297ab99e59","task_key":"default--validation--20830","task_revision_id":"1","upstream_id":"20830","short_description":"HellaSwag validation 20830","config":"default","split":"validation","body":"{\"activity_label\":\"Assembling bicycle\",\"ctx\":\"He is holding a new tube for a tire. He grabs an air pump and attaches it to the tire. he\",\"endings\":\"['grabs the heat pump and attaches it onto the rubber tube.', 'replaces the pump, tightens it and attaches another.', 'removes the tire from the tube gadget.', 'pumps the tire with air.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}