{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"31f5aace-f3c6-5ef0-8773-3708f11fb07b","task_key":"bbeh~5fmovie~5frecommendation--full--31f5aace-f3c6-5ef0-8773-3708f11fb07b","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Dances with Wolves, Cool Runnings, The Shawshank Redemption, Braveheart, The Fugitive\\n(B) Get Shorty, Dances with Wolves, The Fugitive, The Shawshank Redemption, Cool Runnings\\n(C) Cool Runnings, The Fugitive, Braveheart, The Shawshank Redemption, Barbershop 2 Back in Business\\n(D) Braveheart, Get Shorty, Dances with Wolves, The Shawshank Redemption, Barbershop 2 Back in Business\\n(E) To Be or Not to Be, Cool Runnings, Barbershop 2 Back in Business, Braveheart, The Fugitive\\n(F) Barbershop 2 Back in Business, To Be or Not to Be, Get Shorty, The Shawshank Redemption, Braveheart\\n(G) To Be or Not to Be, The Shawshank Redemption, Get Shorty, Barbershop 2 Back in Business, Cool Runnings\\n(H) The Fugitive, Braveheart, The Shawshank Redemption, Get Shorty, Dances with Wolves\\n(I) The Shawshank Redemption, Dances with Wolves, To Be or Not to Be, Barbershop 2 Back in Business, Cool Runnings\\n(J) To Be or Not to Be, Cool Runnings, The Fugitive, Barbershop 2 Back in Business, The Shawshank Redemption\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}