{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1616be09-8e15-555e-9800-371faf911f6f","task_key":"default--validation--16009","task_revision_id":"1","upstream_id":"16009","short_description":"HellaSwag validation 16009","config":"default","split":"validation","body":"{\"activity_label\":\"Installing carpet\",\"ctx\":\"He turns the tool and uses the cutting edge to slice the carpet at the wall seam and tears off the cut strip. he\",\"endings\":\"['removes the bleaches and pulls out the window.', 'turns the tool around and uses the roller end to tuck the carpet in at the wall seam.', 'moves down the wallpaper on the wall to show more work cuts.', 'then cuts the old piece of wallpaper with the scissors.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}