{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"17e7f725-08d5-5fe4-99eb-0d7172a08443","task_key":"default--validation--36563","task_revision_id":"1","upstream_id":"36563","short_description":"HellaSwag validation 36563","config":"default","split":"validation","body":"{\"activity_label\":\"Making a sandwich\",\"ctx\":\"A hand cooked the bacon on the pan. The tomato is being sliced on the board. the mayonnaise\",\"endings\":\"['is being added to the sausage in tongs.', 'is being fried on it.', 'is spread on the toasted bread and then the ham is placed and then cheese, then mayo is spread again, then the bacon is added and then the lettuce and tomatoes.', 'is being shown on a plate with strawberries in it.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}