{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"106bd687-7181-5076-9f9c-3d4c1fed6a07","task_key":"default--validation--49716","task_revision_id":"1","upstream_id":"49716","short_description":"HellaSwag validation 49716","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"A girl dressed in a white blouse and black skirt is standing on stage in front of an orchestra with her violin. the orchestra\",\"endings\":\"['plays the song as the girl tilts her head side to side and stares off into the back of the room.', 'begins playing the violin in a church setting and the girl on stage puts her violin down and stands still while sitting on the piano.', 'quickly starts to form up and begin playing.', 'begins to play and close ups of the clarinets are shown.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}