{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fa9ff539-d386-5ccc-946d-1920c27b1ef4","task_key":"default--validation--20622","task_revision_id":"1","upstream_id":"20622","short_description":"HellaSwag validation 20622","config":"default","split":"validation","body":"{\"activity_label\":\"Ice fishing\",\"ctx\":\"A man is sitting on a bucket ice fishing. He stands up and pulls a fish out of the hole. he\",\"endings\":\"['cuts the hook out of the fish and throws the fish on the ice.', 'pulls it out and spits it out.', 'gets on the fishing line and pulls it to the water.', 'pulls it out of the hole and lays it in the water.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}