{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0652b053-ee35-5110-a852-5b71acf0e476","task_key":"bbeh~5fmovie~5frecommendation--full--0652b053-ee35-5110-a852-5b71acf0e476","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Return with Honor, The Maltese Falcon, One Flew Over the Cuckoo's Nest, The Sixth Sense, In the Loop\\n(B) The Sixth Sense, Return with Honor, Wallace & Gromit A Close Shave, Reservoir Dogs, The Maltese Falcon\\n(C) The Maltese Falcon, The Sixth Sense, One Flew Over the Cuckoo's Nest, Wallace & Gromit A Close Shave, Reservoir Dogs\\n(D) Wallace & Gromit A Close Shave, The Maltese Falcon, One Flew Over the Cuckoo's Nest, Feast, Reservoir Dogs\\n(E) The Sixth Sense, Reservoir Dogs, The Maltese Falcon, Feast, In the Loop\\n(F) Return with Honor, Reservoir Dogs, The Maltese Falcon, Feast, One Flew Over the Cuckoo's Nest\\n(G) Feast, The Maltese Falcon, One Flew Over the Cuckoo's Nest, The Sixth Sense, Reservoir Dogs\\n(H) Return with Honor, In the Loop, Feast, Wallace & Gromit A Close Shave, Reservoir Dogs\\n(I) The Maltese Falcon, Wallace & Gromit A Close Shave, The Sixth Sense, Feast, One Flew Over the Cuckoo's Nest\\n(J) Reservoir Dogs, The Sixth Sense, One Flew Over the Cuckoo's Nest, Feast, Wallace & Gromit A Close Shave\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}