{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7aaa1d7c-3da3-5f99-aeff-e629d41f73ba","task_key":"bbeh~5fmovie~5frecommendation--full--7aaa1d7c-3da3-5f99-aeff-e629d41f73ba","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Fugitive, Crimson Tide, Pulp Fiction, Braveheart, The Shawshank Redemption\\n(B) A Separation, Pulp Fiction, The Shawshank Redemption, Goon Last of the Enforcers, A Very Harold & Kumar 3D Christmas\\n(C) The Shawshank Redemption, Crimson Tide, Braveheart, Pulp Fiction, Goon Last of the Enforcers\\n(D) Braveheart, The Fugitive, Crimson Tide, Pulp Fiction, Goon Last of the Enforcers\\n(E) Goon Last of the Enforcers, A Separation, Pulp Fiction, Braveheart, The Shawshank Redemption\\n(F) Goon Last of the Enforcers, A Very Harold & Kumar 3D Christmas, Crimson Tide, Braveheart, A Separation\\n(G) A Separation, The Shawshank Redemption, Braveheart, A Very Harold & Kumar 3D Christmas, Goon Last of the Enforcers\\n(H) Crimson Tide, The Shawshank Redemption, Goon Last of the Enforcers, The Fugitive, Braveheart\\n(I) Crimson Tide, The Fugitive, Pulp Fiction, A Very Harold & Kumar 3D Christmas, The Shawshank Redemption\\n(J) Pulp Fiction, The Shawshank Redemption, Goon Last of the Enforcers, Crimson Tide, The Fugitive\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}