{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"12efdc7d-eea9-5688-aabb-0367124d778f","task_key":"default--validation--47438","task_revision_id":"1","upstream_id":"47438","short_description":"HellaSwag validation 47438","config":"default","split":"validation","body":"{\"activity_label\":\"Playing harmonica\",\"ctx\":\"A man takes a seat in a chair and raises up his harmonica. An man play a song on his harmonica seated at home. the man\",\"endings\":\"['stops playing and looks away.', 'looks the camera and smiles.', 'vigorously moves the harmonica back and forth over the entire range of notes from going from one end immediately back to the other.', 'stops playing and stands up looking at the camera.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}