{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0d90a4e1-1033-5f49-8a09-400fa0a65138","task_key":"default--validation--11287","task_revision_id":"1","upstream_id":"11287","short_description":"HellaSwag validation 11287","config":"default","split":"validation","body":"{\"activity_label\":\"Gargling mouthwash\",\"ctx\":\"A girl in an orange shirt takes a drink of mouth wash. she\",\"endings\":\"['wipes her mouth down with the back of her hand.', 'throws it back in front of her and then continues smoking.', 'swishes it around in her mouth and spits it out into a sink.', 'spits the mouthwash on the ground.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}