{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"606a27db-0fe8-5e17-bfd5-67d7b4a61d7b","task_key":"default--validation--14141","task_revision_id":"1","upstream_id":"14141","short_description":"HellaSwag validation 14141","config":"default","split":"validation","body":"{\"activity_label\":\"Arm wrestling\",\"ctx\":\"A shot is seen of two men performing an arm wrestling match together while the audience cheers. one man\",\"endings\":\"['wins by curlew flips and the audience applauds.', 'beats the other and the two hold up their arms to cheer and everybody celebrates.', \\\"holds the other man's other arm while the competitors continue fighting around each other.\\\", 'is then shown put down and the third man succeeds in the match.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}