{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a412a4a4-cdce-55b9-b480-7c58da08a9e8","task_key":"default--validation--7507","task_revision_id":"1","upstream_id":"7507","short_description":"HellaSwag validation 7507","config":"default","split":"validation","body":"{\"activity_label\":\"Cleaning shoes\",\"ctx\":\"A person is sitting down holding blue shoes. They hold a bottle wet wipes. they\",\"endings\":\"['rub the blue spots on the shoes.', 'use the wet wipes to clean several rose petals off their shoes.', 'start wiping all over their feet.', 'take one and start wiping the bottom of the shoe.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}