{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b18a9366-00b4-5c56-842d-4ca29ae4cefe","task_key":"bbeh~5fmovie~5frecommendation--full--b18a9366-00b4-5c56-842d-4ca29ae4cefe","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Reservoir Dogs, Raiders of the Lost Ark, One Flew Over the Cuckoo's Nest, Star Wars Episode V - The Empire Strikes Back, Léon The Professional\\n(B) One Flew Over the Cuckoo's Nest, The Thinning, Star Wars Episode V - The Empire Strikes Back, Hereafter, Reservoir Dogs\\n(C) Hereafter, Star Wars Episode V - The Empire Strikes Back, Reservoir Dogs, One Flew Over the Cuckoo's Nest, Léon The Professional\\n(D) Reservoir Dogs, Hereafter, The Thinning, Thumbelina, Raiders of the Lost Ark\\n(E) Star Wars Episode V - The Empire Strikes Back, The Thinning, Thumbelina, Léon The Professional, Hereafter\\n(F) Star Wars Episode V - The Empire Strikes Back, Raiders of the Lost Ark, The Thinning, Reservoir Dogs, Thumbelina\\n(G) Léon The Professional, The Thinning, One Flew Over the Cuckoo's Nest, Reservoir Dogs, Raiders of the Lost Ark\\n(H) Léon The Professional, The Thinning, Hereafter, One Flew Over the Cuckoo's Nest, Thumbelina\\n(I) Star Wars Episode V - The Empire Strikes Back, The Thinning, Léon The Professional, Hereafter, Reservoir Dogs\\n(J) Raiders of the Lost Ark, Thumbelina, The Thinning, Hereafter, Léon The Professional\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}