{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"567804eb-49d1-57e0-be07-37a9666cc52c","task_key":"default--validation--6840","task_revision_id":"1","upstream_id":"6840","short_description":"HellaSwag validation 6840","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"A man is seen speaking to another holding a violin. the man speaking them\",\"endings\":\"['on the violin while the woman plays the violin.', 'back and fourth while the man hits the strings.', 'walks out of frame.', 'back and fourth while he dances to polka music in the end.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}