{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4a90addd-ff07-5b09-9a06-305987ec5f48","task_key":"bbeh~5fmovie~5frecommendation--full--4a90addd-ff07-5b09-9a06-305987ec5f48","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Forrest Gump, Dumb & Dumber, Braveheart, Jurassic Park, Gone with the Wind\\n(B) Gone with the Wind, The Counterfeiters, Forrest Gump, Schindler's List, Pitfall\\n(C) Schindler's List, Jurassic Park, Pitfall, Gone with the Wind, The Counterfeiters\\n(D) Braveheart, Jurassic Park, Gone with the Wind, Schindler's List, Forrest Gump\\n(E) Forrest Gump, The Counterfeiters, Jurassic Park, Schindler's List, Dumb & Dumber\\n(F) Jurassic Park, Dumb & Dumber, Forrest Gump, Braveheart, Schindler's List\\n(G) Pitfall, Dumb & Dumber, Forrest Gump, Schindler's List, Braveheart\\n(H) Dumb & Dumber, Pitfall, Braveheart, The Counterfeiters, Forrest Gump\\n(I) Gone with the Wind, Schindler's List, Pitfall, Dumb & Dumber, Forrest Gump\\n(J) Gone with the Wind, Schindler's List, The Counterfeiters, Braveheart, Jurassic Park\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}