{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bc0016b1-57b1-57cd-b15d-43213593bb9e","task_key":"bbeh~5fmovie~5frecommendation--full--bc0016b1-57b1-57cd-b15d-43213593bb9e","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Elizabethtown, Forrest Gump, The Fugitive, The Bride of Frankenstein, All Over the Guy\\n(B) Forrest Gump, Dances with Wolves, Pulp Fiction, All Over the Guy, The Bride of Frankenstein\\n(C) Pulp Fiction, All Over the Guy, Elizabethtown, The Bride of Frankenstein, Dances with Wolves\\n(D) Pulp Fiction, The Shawshank Redemption, Elizabethtown, Forrest Gump, The Fugitive\\n(E) Dances with Wolves, Forrest Gump, The Fugitive, Pulp Fiction, The Shawshank Redemption\\n(F) All Over the Guy, The Fugitive, Dances with Wolves, The Shawshank Redemption, Elizabethtown\\n(G) The Bride of Frankenstein, The Fugitive, Forrest Gump, All Over the Guy, Dances with Wolves\\n(H) Forrest Gump, All Over the Guy, Elizabethtown, The Shawshank Redemption, Pulp Fiction\\n(I) Forrest Gump, The Bride of Frankenstein, Dances with Wolves, Pulp Fiction, The Fugitive\\n(J) The Shawshank Redemption, Forrest Gump, Elizabethtown, Dances with Wolves, The Fugitive\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}