{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b8eb8cf0-9583-5339-bb5a-868c53acea73","task_key":"default--validation--4089","task_revision_id":"1","upstream_id":"4089","short_description":"HellaSwag validation 4089","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"A man comes onto the screen with a violin and begins to explain different techniques for playing the violin. he\",\"endings\":\"['picks up the violin and begins to demonstrate how to play a song on it.', 'briefly shows off the violin and the different ways you can do it.', 'moves closer to the camera so that you can see the detail of what he is doing with his hands.', 'continues to explain what he will be doing as eventually the video stops and ends with the closing credits shown on the screen.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}