{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"3c1e8328-7759-578d-a12c-a96ea5a84a9d","task_key":"bbeh~5fmovie~5frecommendation--full--3c1e8328-7759-578d-a12c-a96ea5a84a9d","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Dark Knight, Those Magnificent Men in Their Flying Machines, Friday After Next, Couples Retreat, Schindler's List\\n(B) Star Wars Episode IV - A New Hope, Schindler's List, The Dark Knight, Inception, The Lord of the Rings The Two Towers\\n(C) Schindler's List, Couples Retreat, Those Magnificent Men in Their Flying Machines, Inception, Friday After Next\\n(D) Those Magnificent Men in Their Flying Machines, Schindler's List, Couples Retreat, Friday After Next, Star Wars Episode IV - A New Hope\\n(E) The Lord of the Rings The Two Towers, Friday After Next, Those Magnificent Men in Their Flying Machines, Star Wars Episode IV - A New Hope, Schindler's List\\n(F) The Dark Knight, The Lord of the Rings The Two Towers, Schindler's List, Friday After Next, Couples Retreat\\n(G) The Dark Knight, Couples Retreat, Friday After Next, Those Magnificent Men in Their Flying Machines, Star Wars Episode IV - A New Hope\\n(H) Those Magnificent Men in Their Flying Machines, Inception, The Dark Knight, Schindler's List, Star Wars Episode IV - A New Hope\\n(I) Star Wars Episode IV - A New Hope, Inception, Those Magnificent Men in Their Flying Machines, Friday After Next, Schindler's List\\n(J) Schindler's List, Couples Retreat, Friday After Next, The Dark Knight, Inception\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}