{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"312107b4-afeb-5066-9240-8e855c8df54c","task_key":"bbeh~5fmovie~5frecommendation--full--312107b4-afeb-5066-9240-8e855c8df54c","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Star Wars Episode V - The Empire Strikes Back, The Adventures of Sherlock Holmes and Doctor Watson, Forrest Gump, The Lord of the Rings The Two Towers, American Beauty\\n(B) The Adventures of Sherlock Holmes and Doctor Watson, Upside Down The Creation Records Story, The Lord of the Rings The Two Towers, Forrest Gump, Waking Life\\n(C) Forrest Gump, American Beauty, Star Wars Episode V - The Empire Strikes Back, The Lord of the Rings The Fellowship of the Ring, The Lord of the Rings The Two Towers\\n(D) American Beauty, Waking Life, The Lord of the Rings The Fellowship of the Ring, The Adventures of Sherlock Holmes and Doctor Watson, Upside Down The Creation Records Story\\n(E) American Beauty, Waking Life, Star Wars Episode V - The Empire Strikes Back, Forrest Gump, The Lord of the Rings The Fellowship of the Ring\\n(F) Star Wars Episode V - The Empire Strikes Back, Forrest Gump, Upside Down The Creation Records Story, The Lord of the Rings The Two Towers, American Beauty\\n(G) Waking Life, The Lord of the Rings The Two Towers, American Beauty, The Adventures of Sherlock Holmes and Doctor Watson, The Lord of the Rings The Fellowship of the Ring\\n(H) The Lord of the Rings The Fellowship of the Ring, The Lord of the Rings The Two Towers, Upside Down The Creation Records Story, Waking Life, Forrest Gump\\n(I) American Beauty, Star Wars Episode V - The Empire Strikes Back, The Lord of the Rings The Fellowship of the Ring, Waking Life, Upside Down The Creation Records Story\\n(J) Upside Down The Creation Records Story, Forrest Gump, The Adventures of Sherlock Holmes and Doctor Watson, Waking Life, American Beauty\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}