{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a9707d8c-422e-5e35-bf4d-eb1754ed9ae0","task_key":"default--validation--2651","task_revision_id":"1","upstream_id":"2651","short_description":"HellaSwag validation 2651","config":"default","split":"validation","body":"{\"activity_label\":\"Hand car wash\",\"ctx\":\"The woman from before who is dressed in an orange and red dress appears again and is still holding the white paper as she talks. flashing words\",\"endings\":\"['appear on the screen announcing it is time to start as the woman speaks again.', 'appear after the woman is talking and continues to the end.', 'at the bottom come back at her and she puts the white paper in the at the bottom and makes an x below it for a while.', 'appear on the screen and they say \\\" curt\\\\'s theme comedy \\\" enacted attached to a fedex limit tape.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}