{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"da850701-a065-539d-8c97-b71fd776d4af","task_key":"default--validation--40475","task_revision_id":"1","upstream_id":"40475","short_description":"HellaSwag validation 40475","config":"default","split":"validation","body":"{\"activity_label\":\"Smoking hookah\",\"ctx\":\"A woman with curly hair and very thing eyebrows is sucking from a hookah. she\",\"endings\":\"['throws up her hands in frustration.', 'blows out all of the smoke that she inhaled, she is laying in bed by herself relaxing.', 'holds it up to the camera, puts her hands in the air and then kes.', \\\"puts the mouthpiece up to her mouth and inhales deeply revealing her mouth's contents.\\\"]\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}