{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"00bb61de-1444-53c2-adcf-46ef2e5e3d26","task_key":"bbeh~5fmovie~5frecommendation--full--00bb61de-1444-53c2-adcf-46ef2e5e3d26","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Schindler's List, Pulp Fiction, Braveheart, Dances with Wolves, The Shawshank Redemption\\n(B) Hot Lead and Cold Feet, Schindler's List, Dances with Wolves, The Shawshank Redemption, Pulp Fiction\\n(C) Schindler's List, The Man Who Shot Liberty Valance, Braveheart, Dances with Wolves, The Shawshank Redemption\\n(D) Schindler's List, Hot Lead and Cold Feet, Dances with Wolves, Braveheart, Exte Hair Extensions\\n(E) Braveheart, Schindler's List, Dances with Wolves, Exte Hair Extensions, Pulp Fiction\\n(F) Hot Lead and Cold Feet, Braveheart, Pulp Fiction, Schindler's List, The Man Who Shot Liberty Valance\\n(G) Dances with Wolves, The Shawshank Redemption, Hot Lead and Cold Feet, Braveheart, Pulp Fiction\\n(H) Braveheart, Pulp Fiction, Dances with Wolves, Hot Lead and Cold Feet, Schindler's List\\n(I) The Shawshank Redemption, Schindler's List, Braveheart, Pulp Fiction, Exte Hair Extensions\\n(J) The Shawshank Redemption, Braveheart, Dances with Wolves, Hot Lead and Cold Feet, The Man Who Shot Liberty Valance\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}