{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"5a6d0377-5451-5e3a-b361-0cf0aad80f08","task_key":"bbeh~5fmovie~5frecommendation--full--5a6d0377-5451-5e3a-b361-0cf0aad80f08","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Lord of the Rings The Fellowship of the Ring, The End of the Affair, The Lord of the Rings The Two Towers, Star Wars Episode IV - A New Hope, Schindler's List\\n(B) Schindler's List, Star Wars Episode IV - A New Hope, The Lord of the Rings The Fellowship of the Ring, The Return, The Lord of the Rings The Two Towers\\n(C) Star Wars Episode IV - A New Hope, The Matrix, The Lord of the Rings The Fellowship of the Ring, Schindler's List, The Hidden Fortress\\n(D) Schindler's List, Star Wars Episode IV - A New Hope, The Matrix, The Lord of the Rings The Two Towers, The Lord of the Rings The Fellowship of the Ring\\n(E) The Return, The Lord of the Rings The Fellowship of the Ring, The Matrix, Star Wars Episode IV - A New Hope, Schindler's List\\n(F) The Lord of the Rings The Fellowship of the Ring, The End of the Affair, The Hidden Fortress, The Return, Schindler's List\\n(G) The Lord of the Rings The Fellowship of the Ring, Star Wars Episode IV - A New Hope, The End of the Affair, The Hidden Fortress, The Matrix\\n(H) The Matrix, The Lord of the Rings The Fellowship of the Ring, The Hidden Fortress, The End of the Affair, The Return\\n(I) Schindler's List, The End of the Affair, The Lord of the Rings The Two Towers, The Return, The Matrix\\n(J) The Lord of the Rings The Two Towers, The Matrix, Star Wars Episode IV - A New Hope, The Return, The Lord of the Rings The Fellowship of the Ring\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}