{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"26590c8a-5a09-5cd3-972f-db998879343a","task_key":"bbeh~5fmovie~5frecommendation--full--26590c8a-5a09-5cd3-972f-db998879343a","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Hudson Hawk, The Brothers Grimm, Star Wars Episode V - The Empire Strikes Back, The Matrix, Pulp Fiction\\n(B) Pulp Fiction, Hudson Hawk, Star Wars Episode IV - A New Hope, The Brothers Grimm, Star Wars Episode V - The Empire Strikes Back\\n(C) Star Wars Episode IV - A New Hope, Forrest Gump, Star Wars Episode V - The Empire Strikes Back, The Matrix, Pulp Fiction\\n(D) Hudson Hawk, Star Wars Episode IV - A New Hope, Forrest Gump, Star Wars Episode V - The Empire Strikes Back, Pulp Fiction\\n(E) The Matrix, Pulp Fiction, Star Wars Episode V - The Empire Strikes Back, Da geht noch was!, Hudson Hawk\\n(F) Hudson Hawk, The Matrix, Pulp Fiction, The Brothers Grimm, Da geht noch was!\\n(G) Forrest Gump, Star Wars Episode IV - A New Hope, Hudson Hawk, Star Wars Episode V - The Empire Strikes Back, Da geht noch was!\\n(H) Star Wars Episode IV - A New Hope, The Brothers Grimm, Hudson Hawk, Da geht noch was!, The Matrix\\n(I) Hudson Hawk, Pulp Fiction, Forrest Gump, The Brothers Grimm, The Matrix\\n(J) The Brothers Grimm, Pulp Fiction, Forrest Gump, Star Wars Episode IV - A New Hope, Da geht noch was!\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}