{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"314c15fe-d05f-502f-8679-29b4c4a7d0cc","task_key":"default--validation--28803","task_revision_id":"1","upstream_id":"28803","short_description":"HellaSwag validation 28803","config":"default","split":"validation","body":"{\"activity_label\":\"Playing pool\",\"ctx\":\"Professional pool players play a game of billiards in a competition. The man holds out his billiards stick to line up a shot then takes the shot. the man\",\"endings\":\"['holds his shot and makes a shot into the pool.', 'chalks the end of his pool cue and walks to the corner of the table.', 'drains from his glass and sets it down with his hands.', 'takes a shot on his own.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}