{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0f2de73e-bf34-500b-a5b4-392832ec672b","task_key":"default--validation--20829","task_revision_id":"1","upstream_id":"20829","short_description":"HellaSwag validation 20829","config":"default","split":"validation","body":"{\"activity_label\":\"Assembling bicycle\",\"ctx\":\"A man is standing next to a bicycle. He is holding a new tube for a tire. he\",\"endings\":\"['is attaching it to a battery on the bicycle.', 'then turns it on to replace his old one.', 'grabs an air pump and attaches it to the tire.', 'starts cleaning the rust off of the tube.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}