{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bd797193-b515-5437-944f-03472105a9fb","task_key":"bbeh~5fmovie~5frecommendation--full--bd797193-b515-5437-944f-03472105a9fb","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Pretty Woman, Interview with the Vampire The Vampire Chronicles, Batman, Babe, The Mask\\n(B) Bright, Zoom, Unicorn City, Pretty Woman, Babe\\n(C) Unicorn City, Interview with the Vampire The Vampire Chronicles, Babe, Batman, The Mask\\n(D) Unicorn City, The Mask, Pretty Woman, Zoom, Bright\\n(E) Babe, Bright, Zoom, Unicorn City, The Mask\\n(F) The Mask, Batman, Interview with the Vampire The Vampire Chronicles, Bright, Babe\\n(G) The Mask, Unicorn City, Batman, Zoom, Bright\\n(H) The Mask, Zoom, Babe, Pretty Woman, Bright\\n(I) Pretty Woman, Bright, Zoom, Interview with the Vampire The Vampire Chronicles, Unicorn City\\n(J) The Mask, Unicorn City, Bright, Interview with the Vampire The Vampire Chronicles, Pretty Woman\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}