{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8c1d3a8d-01c9-5c0c-bd4e-d89739451cce","task_key":"default--validation--49515","task_revision_id":"1","upstream_id":"49515","short_description":"HellaSwag validation 49515","config":"default","split":"validation","body":"{\"activity_label\":\"Playing harmonica\",\"ctx\":\"A man demonstrates, in a living room setting, how to play the harmonica. a man\",\"endings\":\"['plays a harmonica for the camera attached to a smoking guitar on television set.', 'plays the harmonica for the camera while another man plays to make music for the audience.', 'demonstrates, in slow motion, how to play the harmonica with an elliptical.', 'holds a harmonica up to the camera in a close up shot and then begins to play using both hands to cover the harmonica while he plays.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}