{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b2c5a751-5d4e-5026-8ce2-678d73deddc1","task_key":"default--validation--24224","task_revision_id":"1","upstream_id":"24224","short_description":"HellaSwag validation 24224","config":"default","split":"validation","body":"{\"activity_label\":\"Running a marathon\",\"ctx\":\"A man in black shorts with a number pinned to his chest is running down a city street in a marathon. people on the sidelines\",\"endings\":\"['are watching him as his strides are shown.', 'are clapping for the man and smiling as he continues to run.', 'watching as he runs, trying to catch him.', 'watch him on the sides and cheers behind them.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}