{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"70b3ad4e-4367-52ce-8526-6c46173eb55a","task_key":"default--validation--42416","task_revision_id":"1","upstream_id":"42416","short_description":"HellaSwag validation 42416","config":"default","split":"validation","body":"{\"activity_label\":\"Playing drums\",\"ctx\":\"A singer is shown standing on a piano on a large stage and talking into the mic while a drummer plays in the background. he\",\"endings\":\"['continues playing the piano and ends by shaking hands and walking off stage.', 'continues singing to the audience followed by jumping off the piano and dancing around the stage.', 'continues to play the piano running out to the end and stopping for a while to talk to the audience.', 'begins to play a tune while a close up of the piano is shown.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}