{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4c2cc0d0-28ef-5091-911f-09c8de823faf","task_key":"bbeh~5fmovie~5frecommendation--full--4c2cc0d0-28ef-5091-911f-09c8de823faf","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) War for the Planet of the Apes, Inception, Forrest Gump, Pulp Fiction, The Dark Knight\\n(B) The Dark Knight, Inception, Identity Thief, The Da Vinci Code, Pulp Fiction\\n(C) The Da Vinci Code, The Dark Knight, War for the Planet of the Apes, Identity Thief, Pulp Fiction\\n(D) Inception, The Da Vinci Code, The Silence of the Lambs, Identity Thief, War for the Planet of the Apes\\n(E) Forrest Gump, The Silence of the Lambs, The Da Vinci Code, Inception, The Dark Knight\\n(F) Identity Thief, The Da Vinci Code, Forrest Gump, The Dark Knight, Inception\\n(G) The Silence of the Lambs, Pulp Fiction, Inception, Identity Thief, Forrest Gump\\n(H) Inception, The Dark Knight, Pulp Fiction, The Silence of the Lambs, Forrest Gump\\n(I) War for the Planet of the Apes, The Da Vinci Code, Identity Thief, Forrest Gump, The Dark Knight\\n(J) Identity Thief, Pulp Fiction, Inception, The Dark Knight, Forrest Gump\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}