{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e358ee18-3159-58c7-88de-54284c79afdb","task_key":"default--validation--21287","task_revision_id":"1","upstream_id":"21287","short_description":"HellaSwag validation 21287","config":"default","split":"validation","body":"{\"activity_label\":\"Swimming\",\"ctx\":\"The woman begins to emulate the man's actions, moving her arms and dipping her head under the water. The man shows her how to swim outward by doing so himself. the woman\",\"endings\":\"['finishes and out of teach.', 'watches as he swims away.', 'moves her hands towards the camera as she does so.', 'then sits on the ground.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}