{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e9401759-9b25-5300-bae5-d5b9a5561e57","task_key":"bbeh~5fmovie~5frecommendation--full--e9401759-9b25-5300-bae5-d5b9a5561e57","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Apollo 13, Eichmann, Braveheart, Children of the Night, Kazaam\\n(B) Apollo 13, Braveheart, Eichmann, Kazaam, Schindler's List\\n(C) Children of the Night, Kazaam, The Usual Suspects, Apollo 13, Eichmann\\n(D) Schindler's List, Braveheart, Pulp Fiction, The Usual Suspects, Apollo 13\\n(E) The Usual Suspects, Pulp Fiction, Children of the Night, Braveheart, Apollo 13\\n(F) Apollo 13, Pulp Fiction, The Usual Suspects, Children of the Night, Schindler's List\\n(G) The Usual Suspects, Eichmann, Braveheart, Kazaam, Schindler's List\\n(H) The Usual Suspects, Eichmann, Children of the Night, Pulp Fiction, Kazaam\\n(I) Kazaam, The Usual Suspects, Apollo 13, Pulp Fiction, Braveheart\\n(J) Braveheart, Pulp Fiction, Children of the Night, Kazaam, Eichmann\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}