{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"cc18f3ab-0b92-5640-9d1e-e87f48e79961","task_key":"bbeh~5fmovie~5frecommendation--full--cc18f3ab-0b92-5640-9d1e-e87f48e79961","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Ip Man 3, Star Wars Episode V - The Empire Strikes Back, The Wedding Ringer, Delivery Man, Raiders of the Lost Ark\\n(B) The Silence of the Lambs, Raiders of the Lost Ark, Star Wars Episode IV - A New Hope, The Wedding Ringer, Delivery Man\\n(C) Delivery Man, Star Wars Episode IV - A New Hope, The Wedding Ringer, Ip Man 3, The Silence of the Lambs\\n(D) Goodfellas, Star Wars Episode IV - A New Hope, The Silence of the Lambs, Delivery Man, Ip Man 3\\n(E) Delivery Man, Ip Man 3, Star Wars Episode V - The Empire Strikes Back, The Wedding Ringer, Star Wars Episode IV - A New Hope\\n(F) Raiders of the Lost Ark, Star Wars Episode V - The Empire Strikes Back, The Silence of the Lambs, Goodfellas, Star Wars Episode IV - A New Hope\\n(G) Star Wars Episode V - The Empire Strikes Back, The Silence of the Lambs, Raiders of the Lost Ark, Delivery Man, The Wedding Ringer\\n(H) The Silence of the Lambs, Ip Man 3, Raiders of the Lost Ark, Delivery Man, The Wedding Ringer\\n(I) Raiders of the Lost Ark, Star Wars Episode V - The Empire Strikes Back, Delivery Man, Ip Man 3, Star Wars Episode IV - A New Hope\\n(J) Ip Man 3, Star Wars Episode IV - A New Hope, Raiders of the Lost Ark, Star Wars Episode V - The Empire Strikes Back, The Silence of the Lambs\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}