{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"hellaswag","formal_name":"HellaSwag","introduction":"HellaSwag asks which of four continuations naturally follows a sentence or a video caption. The wrong options are produced by adversarial filtering, so items that are nearly trivial for people remain hard for models.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://rowanzellers.com/hellaswag/","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"48628d48-7cb5-549f-a8bc-92290720400b","task_key":"default--validation--49230","task_revision_id":"1","upstream_id":"49230","short_description":"HellaSwag validation 49230","config":"default","split":"validation","body":"{\"activity_label\":\"Making a lemonade\",\"ctx\":\"They skin lemons and set them on the plate. They put the fruit into a juicer and pour the juice into a large jug. they\",\"endings\":\"['mix sugar and water together together and pour the juice into the jug.', 'mash the lemons and put it on the plate.', 'shake the jug in their hands.', 'pour the syrup into a tuber cup and pour the syrup over this, then they pour the syrup from the jug into a bottle.']\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://rowanzellers.com/hellaswag/","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}