{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3de07c7c-e68d-55e4-9324-4e2dd36dd79e","task_key":"default--validation--37672","task_revision_id":"1","upstream_id":"37672","short_description":"HellaSwag validation 37672","config":"default","split":"validation","body":"{\"activity_label\":\"Gargling mouthwash\",\"ctx\":\"Two girls are seen speaking to the camera and holding up glasses. They clink and hold up mouthwash and take a drink from the glass. they\",\"endings\":\"['drink the various liquids together and throw the objects into a sink followed by smiling at the camera.', 'sip the drink and continue to speak to the camera.', 'drink all of the mouthwash and place the glass next to them.', 'swish around the mouthwash and spit it out while laughing.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}