{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f9fae542-66cc-53fd-a463-c4a98fecad66","task_key":"bbeh~5fmovie~5frecommendation--full--f9fae542-66cc-53fd-a463-c4a98fecad66","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Joe's Apartment, Pulp Fiction, Forrest Gump, Raiders of the Lost Ark, Inception\\n(B) Raiders of the Lost Ark, Firestarter, Joe's Apartment, Trail of the Pink Panther, Inception\\n(C) Pulp Fiction, The Matrix, Forrest Gump, Inception, Firestarter\\n(D) The Matrix, Joe's Apartment, Inception, Forrest Gump, Raiders of the Lost Ark\\n(E) Trail of the Pink Panther, Joe's Apartment, Raiders of the Lost Ark, The Matrix, Pulp Fiction\\n(F) Trail of the Pink Panther, The Matrix, Raiders of the Lost Ark, Joe's Apartment, Inception\\n(G) Raiders of the Lost Ark, Pulp Fiction, Forrest Gump, The Matrix, Inception\\n(H) Joe's Apartment, Trail of the Pink Panther, The Matrix, Firestarter, Inception\\n(I) Trail of the Pink Panther, The Matrix, Firestarter, Pulp Fiction, Joe's Apartment\\n(J) The Matrix, Pulp Fiction, Forrest Gump, Inception, Joe's Apartment\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}