{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ce107a82-9886-5dea-988d-1031ffc9e65b","task_key":"default--validation--7733","task_revision_id":"1","upstream_id":"7733","short_description":"HellaSwag validation 7733","config":"default","split":"validation","body":"{\"activity_label\":\"Playing drums\",\"ctx\":\"A young man wearing a hat is seen playing a set of drums in front of an audience while they record him. more people\",\"endings\":\"['are seen playing guitars and singing along while the man continues playing drums along with the song.', 'are seen performing with him playing the drums and being interviewed by the scene.', 'walk in and out of frame while he plays, pausing for a side view and others getting closer.', 'appear as the continues to play with one another and ends with the man embracing others.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}