{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"04a23cd3-f9ad-5521-8d3f-0543fc772a9a","task_key":"default--validation--20623","task_revision_id":"1","upstream_id":"20623","short_description":"HellaSwag validation 20623","config":"default","split":"validation","body":"{\"activity_label\":\"Ice fishing\",\"ctx\":\"He stands up and pulls a fish out of the hole. He cuts the hook out of the fish and throws the fish on the ice. he\",\"endings\":\"['gets back on his hands and helps pull up snow off the ice.', 'raises his hands in the air.', 'gets another fish out of the hole and pulls it out laying in the water.', 'sits back down on the bucket.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}