{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"49565575-7027-546b-a650-e6c898015ed3","task_key":"default--validation--10338","task_revision_id":"1","upstream_id":"10338","short_description":"HellaSwag validation 10338","config":"default","split":"validation","body":"{\"activity_label\":\"Playing harmonica\",\"ctx\":\"A man is seated on a couch, playing a harmonica. he\",\"endings\":\"['lowers it from his mouth when he is done.', 'is playing the acoustic guitar with his eyes closed.', 'cuts his fingers and plays intensely as he plays on the harmonica.', 'switches sides quickly, continuing to play the instrument.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}