{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"57383b2b-0790-5c9c-bffb-b4ad8da06859","task_key":"bbeh~5fmovie~5frecommendation--full--57383b2b-0790-5c9c-bffb-b4ad8da06859","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Ewok Adventure, Changing Lanes, The Matrix, Pulp Fiction, Inception\\n(B) Pulp Fiction, Raiders of the Lost Ark, The Matrix, Forrest Gump, The Ewok Adventure\\n(C) High Noon, Raiders of the Lost Ark, Inception, Pulp Fiction, Forrest Gump\\n(D) Raiders of the Lost Ark, Changing Lanes, High Noon, The Matrix, The Ewok Adventure\\n(E) The Matrix, Inception, The Ewok Adventure, Changing Lanes, High Noon\\n(F) Pulp Fiction, Changing Lanes, Raiders of the Lost Ark, Forrest Gump, Inception\\n(G) Pulp Fiction, The Matrix, Raiders of the Lost Ark, Inception, Forrest Gump\\n(H) The Matrix, The Ewok Adventure, Pulp Fiction, High Noon, Changing Lanes\\n(I) Raiders of the Lost Ark, High Noon, Changing Lanes, The Ewok Adventure, Forrest Gump\\n(J) The Ewok Adventure, Inception, Changing Lanes, Forrest Gump, High Noon\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}