{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"33f44b8e-4fc9-5ec6-960b-c38ddb86a6af","task_key":"default--validation--26191","task_revision_id":"1","upstream_id":"26191","short_description":"HellaSwag validation 26191","config":"default","split":"validation","body":"{\"activity_label\":\"Disc dog\",\"ctx\":\"The man throws frisbees for the dog to catch. The man performs special tricks with the dog using the frisbee. the man\",\"endings\":\"['spins the dog in his hands and catches the frisbees and throw with the dog.', 'jumps on the dog with the frisbee.', 'lifts the dog triumphantly over his head and carries him on his shoulder.', 'jumps on the dog on his back.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}