{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"de8f73ce-3348-50d2-8326-27148a3f1482","task_key":"bbeh~5fmovie~5frecommendation--full--de8f73ce-3348-50d2-8326-27148a3f1482","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Indiana Jones and the Last Crusade, Star Wars Episode V - The Empire Strikes Back, Mission Impossible, Raiders of the Lost Ark, Star Wars Episode IV - A New Hope\\n(B) Star Wars Episode V - The Empire Strikes Back, Mission Impossible, Indiana Jones and the Last Crusade, Raiders of the Lost Ark, Too Late for Tears\\n(C) Mission Impossible, Too Late for Tears, Tower Heist, Raiders of the Lost Ark, Stander\\n(D) Indiana Jones and the Last Crusade, Stander, Too Late for Tears, Tower Heist, Raiders of the Lost Ark\\n(E) Stander, Mission Impossible, Indiana Jones and the Last Crusade, Star Wars Episode V - The Empire Strikes Back, Tower Heist\\n(F) Star Wars Episode V - The Empire Strikes Back, Mission Impossible, Star Wars Episode IV - A New Hope, Too Late for Tears, Indiana Jones and the Last Crusade\\n(G) Star Wars Episode V - The Empire Strikes Back, Tower Heist, Raiders of the Lost Ark, Too Late for Tears, Indiana Jones and the Last Crusade\\n(H) Star Wars Episode V - The Empire Strikes Back, Tower Heist, Stander, Raiders of the Lost Ark, Mission Impossible\\n(I) Tower Heist, Star Wars Episode V - The Empire Strikes Back, Too Late for Tears, Star Wars Episode IV - A New Hope, Raiders of the Lost Ark\\n(J) Too Late for Tears, Raiders of the Lost Ark, Tower Heist, Mission Impossible, Indiana Jones and the Last Crusade\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}