{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"328e1cd4-4131-53d4-8332-3abf677276a8","task_key":"bbeh~5fmovie~5frecommendation--full--328e1cd4-4131-53d4-8332-3abf677276a8","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Crimson Tide, Best Men, A Very Harold & Kumar 3D Christmas, The Shawshank Redemption, The Usual Suspects\\n(B) The Shawshank Redemption, 3000 Miles to Graceland, A Very Harold & Kumar 3D Christmas, Crimson Tide, Best Men\\n(C) Crimson Tide, The Shawshank Redemption, Braveheart, The Usual Suspects, Dances with Wolves\\n(D) A Very Harold & Kumar 3D Christmas, Best Men, Crimson Tide, The Usual Suspects, Dances with Wolves\\n(E) 3000 Miles to Graceland, The Shawshank Redemption, Best Men, Crimson Tide, Braveheart\\n(F) Braveheart, Best Men, Dances with Wolves, Crimson Tide, A Very Harold & Kumar 3D Christmas\\n(G) Dances with Wolves, 3000 Miles to Graceland, The Usual Suspects, The Shawshank Redemption, Braveheart\\n(H) Best Men, Crimson Tide, The Usual Suspects, 3000 Miles to Graceland, Braveheart\\n(I) A Very Harold & Kumar 3D Christmas, Crimson Tide, The Usual Suspects, Braveheart, 3000 Miles to Graceland\\n(J) A Very Harold & Kumar 3D Christmas, 3000 Miles to Graceland, Best Men, The Usual Suspects, Braveheart\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}