{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f9fc1cb3-828b-5f95-8751-ed4bb3e26070","task_key":"default--validation--10659","task_revision_id":"1","upstream_id":"10659","short_description":"HellaSwag validation 10659","config":"default","split":"validation","body":"{\"activity_label\":\"Smoking a cigarette\",\"ctx\":\"A man's face is seen close up speaking and leads into him smoking a cigarette. the man\",\"endings\":\"['then plays a little piano and sits down to smoke away while continuing to speak and play.', 'continues to take drags out of the cigarette while blowing smoke off into the distance.', 'continues smoking with his mouth open and leads into him getting up and speaking to another man as well as smoking cigarettes in a conga line.', 'walls the room with smoke while moving his hands and speaking.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}