{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"722e5938-fb43-527e-b8ad-a31e42b25a60","task_key":"bbeh~5fmovie~5frecommendation--full--722e5938-fb43-527e-b8ad-a31e42b25a60","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Pinocchio, Beauty and the Beast, The Lion King, Snow White and the Seven Dwarfs, The Thing\\n(B) The Lion King, Snow White and the Seven Dwarfs, The Secret Garden, The Thing, Beauty and the Beast\\n(C) Pinocchio, Nymphomaniac Volume I, Beauty and the Beast, Snow White and the Seven Dwarfs, The Lion King\\n(D) Snow White and the Seven Dwarfs, Pinocchio, The Thing, Lights in the Dusk, Nymphomaniac Volume I\\n(E) The Secret Garden, Nymphomaniac Volume I, Snow White and the Seven Dwarfs, The Thing, Lights in the Dusk\\n(F) Snow White and the Seven Dwarfs, Nymphomaniac Volume I, The Thing, Pinocchio, The Lion King\\n(G) Beauty and the Beast, Lights in the Dusk, The Secret Garden, Pinocchio, Snow White and the Seven Dwarfs\\n(H) The Lion King, Pinocchio, The Secret Garden, The Thing, Lights in the Dusk\\n(I) Nymphomaniac Volume I, Beauty and the Beast, Lights in the Dusk, The Thing, Pinocchio\\n(J) Pinocchio, The Lion King, Snow White and the Seven Dwarfs, The Secret Garden, Beauty and the Beast\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}