{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"5d3c1429-60f2-591e-aa5b-737f2667ad28","task_key":"default--validation--4088","task_revision_id":"1","upstream_id":"4088","short_description":"HellaSwag validation 4088","config":"default","split":"validation","body":"{\"activity_label\":\"Playing violin\",\"ctx\":\"An introduction comes onto the screen for a video about how to play different songs on the violin. a man\",\"endings\":\"['comes onto the screen with a violin and begins to explain different techniques for playing the violin.', 'is shown playing the saxophone in his home.', 'is sitting down on the right playing on his acoustic guitar and talking to the camera.', 'with ear protection is shown playing and changing notes.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}