{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ce38bb41-769b-53a9-8d6d-2957d9d73c37","task_key":"bbeh~5fmovie~5frecommendation--full--ce38bb41-769b-53a9-8d6d-2957d9d73c37","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Up, Raiders of the Lost Ark, The Matrix, Star Wars Episode IV - A New Hope, Inception\\n(B) White Christmas, Up, Ernest Rides Again, Inception, The Matrix\\n(C) White Christmas, Ernest Rides Again, The Matrix, Attack of the Killer Tomatoes!, Inception\\n(D) Attack of the Killer Tomatoes!, Star Wars Episode IV - A New Hope, White Christmas, The Matrix, Ernest Rides Again\\n(E) Ernest Rides Again, Raiders of the Lost Ark, White Christmas, Star Wars Episode IV - A New Hope, Inception\\n(F) Ernest Rides Again, Star Wars Episode IV - A New Hope, Up, Inception, White Christmas\\n(G) Star Wars Episode IV - A New Hope, Ernest Rides Again, Up, Raiders of the Lost Ark, Inception\\n(H) Star Wars Episode IV - A New Hope, Attack of the Killer Tomatoes!, Ernest Rides Again, The Matrix, Up\\n(I) Inception, Attack of the Killer Tomatoes!, Ernest Rides Again, White Christmas, Star Wars Episode IV - A New Hope\\n(J) Raiders of the Lost Ark, Ernest Rides Again, Inception, Up, Attack of the Killer Tomatoes!\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}