{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e70e8d4c-9c71-5296-a819-73b0a14eefff","task_key":"default--validation--39577","task_revision_id":"1","upstream_id":"39577","short_description":"HellaSwag validation 39577","config":"default","split":"validation","body":"{\"activity_label\":\"Playing harmonica\",\"ctx\":\"A man sits down with a harmonica on his neck and plays a guitar. he\",\"endings\":\"['takes the harmonica off his neck, adjusts the instrument and picks up another harmonica.', 'stops playing then takes the harmonica out of his mouth to speak.', 'is then shown playing the harmonica by himself on a courtyard of a city park.', 'starts playing the harmonica while still playing the guitar.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}