{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"461fe1be-be2a-5620-9c5c-60d75286fd1f","task_key":"default--validation--3158","task_revision_id":"1","upstream_id":"3158","short_description":"HellaSwag validation 3158","config":"default","split":"validation","body":"{\"activity_label\":\"Spread mulch\",\"ctx\":\"A woman walks over to a flower bed and wheelbarrow as she talks. she\",\"endings\":\"['picks up several bags of flowers on top of the flower bed.', 'demonstrates how to mulch, picking up the material and showing how to pack it around plants.', 'then shovels the dirt around the trunk of a large tree.', 'places the wheelbarrow and supplies inside.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}