{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"757fa4d3-8f43-5c20-8260-7d02fce7b2ab","task_key":"default--validation--4477","task_revision_id":"1","upstream_id":"4477","short_description":"HellaSwag validation 4477","config":"default","split":"validation","body":"{\"activity_label\":\"Having an ice cream\",\"ctx\":\"A man is first seen speaking to the camera with text questions being asked to him. He is then seen putting a cleaner on his teeth and taking a bit of food. he\",\"endings\":\"['scrubs the dish vigorously in an attempt to get rid of the food in there.', 'continues speaking to the camera while occasionally eating food.', 'then smokes some smoke while looking off wildly into the distance.', 'eventually is seen applying a gel and being shaved.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}