{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3f263b33-f0b0-56e5-9148-bcf308736fa2","task_key":"default--validation--33439","task_revision_id":"1","upstream_id":"33439","short_description":"HellaSwag validation 33439","config":"default","split":"validation","body":"{\"activity_label\":\"Playing pool\",\"ctx\":\"The man and woman continue talking to each other. It flashes back to the pool table. the man and woman\",\"endings\":\"['continue talking and the woman laughs at what the man is saying.', 'laugh about capoeira and the lady tosses the ball.', 'joke with one another on the table.', 'are standing with the help of referee.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}