{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6a45f8c1-2b5a-5c47-9c58-5d03b2807b70","task_key":"default--validation--36048","task_revision_id":"1","upstream_id":"36048","short_description":"HellaSwag validation 36048","config":"default","split":"validation","body":"{\"activity_label\":\"Sharpening knives\",\"ctx\":\"He cleans the knife on a towel and shows the residue left behind. He explains the difference between a honing steel and a sharpening steel. a black screen\",\"endings\":\"['appears with blue and white drawings of swords and arrows on it and the word \\\" swords lacrosse \\\" on it.', 'appears on black, then black again.', 'appears and fades into a white screen with online culinary schools, websites and a copyright to a company.', 'flashes and the new recipe appears.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}