{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d5b63cb0-0195-5674-bfe3-dbce284543c2","task_key":"default--validation--36891","task_revision_id":"1","upstream_id":"36891","short_description":"HellaSwag validation 36891","config":"default","split":"validation","body":"{\"activity_label\":\"Shuffleboard\",\"ctx\":\"The lady taps the puck with her stick before pushing it hard and pushing the puck across the court. the camera\",\"endings\":\"['gets itself angled to keep the puck from hitting the boy playing while playground equipment sits nearby.', 'zooms in on the puck and two women are standing next to it talking to each other.', 'zooms in on cupcake detail while the lady takes a breath.', 'follows the pucks which stops short and teeters before resting.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}