{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"93a6821e-270c-53c3-a741-6e847d724fca","task_key":"default--validation--31070","task_revision_id":"1","upstream_id":"31070","short_description":"HellaSwag validation 31070","config":"default","split":"validation","body":"{\"activity_label\":\"Playing accordion\",\"ctx\":\"A man is seen sitting in a busy city square playing an accordion while others watch. the man\",\"endings\":\"['continues playing over and over.', 'continues playing the instrument and ends by holding the microphone.', \\\"pauses while playing and looking back to the people's watching him.\\\", 'continues playing as people walk around and notice.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}