{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"59727be9-83bf-52e4-83cb-17e80185a1cc","task_key":"bbeh~5fmovie~5frecommendation--full--59727be9-83bf-52e4-83cb-17e80185a1cc","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Batman Assault on Arkham, Pretty Woman, Apollo 13, The Flamingo Kid, Dances with Wolves\\n(B) Apollo 13, Pretty Woman, The Shawshank Redemption, Pokémon Heroes, Braveheart\\n(C) Pretty Woman, Pokémon Heroes, The Shawshank Redemption, Braveheart, The Flamingo Kid\\n(D) Pokémon Heroes, The Flamingo Kid, Batman Assault on Arkham, Braveheart, The Shawshank Redemption\\n(E) Pokémon Heroes, Batman Assault on Arkham, The Shawshank Redemption, The Flamingo Kid, Apollo 13\\n(F) Pretty Woman, The Flamingo Kid, Dances with Wolves, Pokémon Heroes, Braveheart\\n(G) Braveheart, Dances with Wolves, The Shawshank Redemption, Apollo 13, Pretty Woman\\n(H) The Flamingo Kid, Batman Assault on Arkham, Braveheart, Pokémon Heroes, Pretty Woman\\n(I) Braveheart, The Shawshank Redemption, Batman Assault on Arkham, Pretty Woman, Apollo 13\\n(J) Pretty Woman, Dances with Wolves, Pokémon Heroes, The Shawshank Redemption, Braveheart\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}