{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8b909987-9515-5b9f-93ad-686644adc8cd","task_key":"default--validation--48472","task_revision_id":"1","upstream_id":"48472","short_description":"HellaSwag validation 48472","config":"default","split":"validation","body":"{\"activity_label\":\"Computers and Electronics\",\"ctx\":\"[header] How to disable comments on videos on youtube [title] Navigate to youtube.com. [title] Sign in to your account. [substeps] Click sign in.\",\"endings\":\"['This link should appear inside a blank box. [title] Click allow me to automatically see your videos.', 'If your password has been reset, resetting it will work for you. When you are logged into youtube.com, you should not see an menu showing your username and password.', 'This blue button is located in the very top, right corner of the screen. Click on the box the reads \\\" enter your email \\\" and type your google email address.', 'Select the email account you want to remove comments from. Click on the link for the email address you want to remove.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}