{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"cc15fd18-7688-58af-aa99-be2f22eff911","task_key":"default--validation--39183","task_revision_id":"1","upstream_id":"39183","short_description":"HellaSwag validation 39183","config":"default","split":"validation","body":"{\"activity_label\":\"Shuffleboard\",\"ctx\":\"We see a girl walking slowly with a shuffleboard pole and pushing a puck. the lady\",\"endings\":\"['makes a hand gesture and the boy walks past us.', 'gets to the blue line and pushes the puck.', 'is standing and carrying the small little giraffe.', 'stops pushing the puck and pushes a key across the floor.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}