{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c305a994-f557-5a1c-a128-e08bc160236b","task_key":"bbeh~5fmovie~5frecommendation--full--c305a994-f557-5a1c-a128-e08bc160236b","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Terminator, Opera, Highlander The Search for Vengeance, Superman, Austin Powers International Man of Mystery\\n(B) Opera, The Terminator, Highlander The Search for Vengeance, Superman, Supergirl\\n(C) Star Wars Episode IV - A New Hope, Highlander The Search for Vengeance, American Beauty, Austin Powers International Man of Mystery, Superman\\n(D) Supergirl, Star Wars Episode IV - A New Hope, Superman, Austin Powers International Man of Mystery, American Beauty\\n(E) Star Wars Episode IV - A New Hope, Superman, American Beauty, Supergirl, The Terminator\\n(F) The Terminator, American Beauty, Highlander The Search for Vengeance, Supergirl, Opera\\n(G) Supergirl, Austin Powers International Man of Mystery, Opera, Superman, Highlander The Search for Vengeance\\n(H) American Beauty, Superman, The Terminator, Supergirl, Opera\\n(I) The Terminator, Star Wars Episode IV - A New Hope, Austin Powers International Man of Mystery, American Beauty, Superman\\n(J) The Terminator, Star Wars Episode IV - A New Hope, Supergirl, Austin Powers International Man of Mystery, Highlander The Search for Vengeance\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}