{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d493f38c-78d1-546b-9e79-97995c2680de","task_key":"bbeh~5fmovie~5frecommendation--full--d493f38c-78d1-546b-9e79-97995c2680de","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) 12 Angry Men, Mindhunters, Braveheart, Mo' Better Blues, The Usual Suspects\\n(B) Braveheart, The Usual Suspects, Pulp Fiction, The Shawshank Redemption, Schindler's List\\n(C) Mindhunters, 12 Angry Men, Pulp Fiction, Mo' Better Blues, The Usual Suspects\\n(D) 12 Angry Men, Schindler's List, Braveheart, Mo' Better Blues, Pulp Fiction\\n(E) The Usual Suspects, Braveheart, 12 Angry Men, Schindler's List, Pulp Fiction\\n(F) Schindler's List, 12 Angry Men, The Shawshank Redemption, The Usual Suspects, Mindhunters\\n(G) Braveheart, 12 Angry Men, Pulp Fiction, Mindhunters, Schindler's List\\n(H) Pulp Fiction, The Shawshank Redemption, The Usual Suspects, Schindler's List, Mo' Better Blues\\n(I) Mo' Better Blues, The Shawshank Redemption, Braveheart, 12 Angry Men, Mindhunters\\n(J) Mo' Better Blues, Braveheart, Mindhunters, Schindler's List, 12 Angry Men\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}