{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"11e564d9-537f-5f87-b508-a6792c2cd1b9","task_key":"default--validation--16267","task_revision_id":"1","upstream_id":"16267","short_description":"HellaSwag validation 16267","config":"default","split":"validation","body":"{\"activity_label\":\"Futsal\",\"ctx\":\"An indoor arena appears and a brazil team is competing with a team from japan in a game of indoor soccer. a penalty kick\",\"endings\":\"['is made and the three continue to play and the coach lets them have it for as long as they need to.', 'is thrown and the teams line up to make the strike.', 'bell is accidentally entered to signal the start of the match.', 'starts the game and the brazil teams jumps in joy because they are leading the game with a two to zero score.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}