{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8b38db28-d63b-518c-89f0-bc7160f3cbfa","task_key":"default--validation--36775","task_revision_id":"1","upstream_id":"36775","short_description":"HellaSwag validation 36775","config":"default","split":"validation","body":"{\"activity_label\":\"Vacuuming floor\",\"ctx\":\"Someone is holding a vacuum cleaner and turn it on to clean the floor someone is vacuuming the floor with an orange vacuum cleaner. different ingredients\",\"endings\":\"['are used in various different parts of the vacuum and machine.', 'are shown and the person then takes off the shoe and places it next to the trashcan the vacuuming is done with.', 'are on the floor and someone cleans it.', 'are added all at once.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}