{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"000d807a-a466-526a-8898-e781855857e2","task_key":"bbeh~5fmovie~5frecommendation--full--000d807a-a466-526a-8898-e781855857e2","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Dances with Wolves, The Shawshank Redemption, The Great Gatsby, PK, Casper Meets Wendy\\n(B) The Great Gatsby, The Shawshank Redemption, Mr Holland's Opus, Forrest Gump, Dances with Wolves\\n(C) The Great Gatsby, Dances with Wolves, Casper Meets Wendy, Forrest Gump, PK\\n(D) The Shawshank Redemption, Casper Meets Wendy, Philadelphia, The Great Gatsby, PK\\n(E) Casper Meets Wendy, Forrest Gump, Philadelphia, The Shawshank Redemption, The Great Gatsby\\n(F) Dances with Wolves, Mr Holland's Opus, Philadelphia, PK, The Great Gatsby\\n(G) Forrest Gump, Philadelphia, PK, Dances with Wolves, Casper Meets Wendy\\n(H) PK, Casper Meets Wendy, Philadelphia, Forrest Gump, The Great Gatsby\\n(I) The Great Gatsby, Philadelphia, Dances with Wolves, Mr Holland's Opus, Forrest Gump\\n(J) Mr Holland's Opus, The Shawshank Redemption, Philadelphia, Dances with Wolves, Forrest Gump\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}