{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"61b9852f-1ab9-57da-83e8-2516ed8b6920","task_key":"default--validation--36242","task_revision_id":"1","upstream_id":"36242","short_description":"HellaSwag validation 36242","config":"default","split":"validation","body":"{\"activity_label\":\"Gargling mouthwash\",\"ctx\":\"A woman wearing glasses has her hand on her hip. She starts talking about a bottle of mouthwash. she\",\"endings\":\"['takes the mouthwash out of her mouth and begins talking to the camera.', 'then uses the mouthwash to gargle and rinse in the sink.', 'makes an effort to eat some lettuce in her mouth.', 'pours some into her mouth and ate some.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}