{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"12ff7fc9-9f3c-5f98-879c-a28f62d3bd64","task_key":"default--validation--40474","task_revision_id":"1","upstream_id":"40474","short_description":"HellaSwag validation 40474","config":"default","split":"validation","body":"{\"activity_label\":\"Smoking hookah\",\"ctx\":\". a woman with curly hair and very thing eyebrows\",\"endings\":\"['with sparkly violet lips are demonstrating how to make or destroy lipsticks.', 'is on tv talking and after she speaks, she puts her tongue back in her mouth and continue talking.', 'walks into a classroom with both her boobs.', 'is sucking from a hookah.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}