{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"dcb4f1c2-1372-5ce4-ba75-e4bbb2eb55df","task_key":"default--validation--21052","task_revision_id":"1","upstream_id":"21052","short_description":"HellaSwag validation 21052","config":"default","split":"validation","body":"{\"activity_label\":\"Smoking a cigarette\",\"ctx\":\"A woman is adjusting a camera in a bedroom. She is smoking a cigarette and ashing it into a tray. she\",\"endings\":\"['puts it back in place and spins it around again.', 'blows out the smoke and shows it off in the end.', 'continues smoking from the tray.', 'grinds the cigarette into the tray when she is finished.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}