{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bf65e354-d0fb-56ea-a6ba-2e93a499a326","task_key":"default--validation--15714","task_revision_id":"1","upstream_id":"15714","short_description":"HellaSwag validation 15714","config":"default","split":"validation","body":"{\"activity_label\":\"Starting a campfire\",\"ctx\":\"A middle age man is standing over a pile of wood. the man\",\"endings\":\"['knells down to the wood.', 'then begins poking the wood and hammering it into place.', 'starts shakeling the wood.', 'grabs kindling and materials and begins using the wood on the log to build a fire.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}