{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3746ea2d-4aef-50ef-a34d-031338c9347c","task_key":"bbeh~5fmovie~5frecommendation--full--3746ea2d-4aef-50ef-a34d-031338c9347c","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Batman, Independence Day, Microwave Massacre, The Mask, To Catch a Thief\\n(B) Microwave Massacre, Forrest Gump, Pleasantville, To Catch a Thief, Jurassic Park\\n(C) Forrest Gump, Pleasantville, Jurassic Park, Batman, Independence Day\\n(D) Jurassic Park, Forrest Gump, Pleasantville, The Mask, Independence Day\\n(E) Microwave Massacre, Independence Day, The Mask, Jurassic Park, Pleasantville\\n(F) Forrest Gump, To Catch a Thief, Independence Day, Batman, Pleasantville\\n(G) Batman, The Mask, Forrest Gump, Independence Day, Jurassic Park\\n(H) Pleasantville, Forrest Gump, Batman, To Catch a Thief, The Mask\\n(I) Jurassic Park, Pleasantville, Forrest Gump, The Mask, Microwave Massacre\\n(J) Jurassic Park, Batman, Microwave Massacre, To Catch a Thief, The Mask\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}