{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"44b349a9-6e96-5f83-9c4b-e1ded988d617","task_key":"default--validation--11025","task_revision_id":"1","upstream_id":"11025","short_description":"HellaSwag validation 11025","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"An intro leads into a man holding a violin and playing with his hands. the man\",\"endings\":\"['continues to play and then pauses to speak to the camera while also demonstrating how to properly play.', 'continues playing while turning red and then green lights begin flashing around and music begins to play.', 'finishes tapping the violin to the side while the camera captures him from several angles.', 'continues playing with the violin in front of him and ends by leading the camera around.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}