{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f8444829-c46d-5684-995a-b8f05378e4c8","task_key":"default--validation--16120","task_revision_id":"1","upstream_id":"16120","short_description":"HellaSwag validation 16120","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"Arnold schwarzenegger in a gold shirt is playing the violin on stage. The crowd dances sarcastically to his music. he\",\"endings\":\"['leaves the stage in frustration.', 'drops another string on his violin.', 'finishes playing and the audience applauds intensely.', 'is drunk and wearing an outfit of red knee length pants and black top.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}