{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"edafb098-4bf5-5757-a322-ea0f64ed3f0d","task_key":"bbeh~5fmovie~5frecommendation--full--edafb098-4bf5-5757-a322-ea0f64ed3f0d","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Run Lola Run, The Big Lebowski, Lightning in a Bottle, Dragon Ball Z Dead Zone, The Omega Man\\n(B) Being John Malkovich, Dragon Ball Z Dead Zone, Goodfellas, The Omega Man, The Wizard of Oz\\n(C) Goodfellas, Dragon Ball Z Dead Zone, The Big Lebowski, The Omega Man, The Wizard of Oz\\n(D) The Big Lebowski, Goodfellas, Being John Malkovich, The Wizard of Oz, Run Lola Run\\n(E) Being John Malkovich, The Wizard of Oz, Goodfellas, Run Lola Run, Lightning in a Bottle\\n(F) Being John Malkovich, Run Lola Run, The Wizard of Oz, Dragon Ball Z Dead Zone, Goodfellas\\n(G) The Omega Man, The Big Lebowski, Dragon Ball Z Dead Zone, Lightning in a Bottle, The Wizard of Oz\\n(H) The Omega Man, Lightning in a Bottle, Dragon Ball Z Dead Zone, The Big Lebowski, Goodfellas\\n(I) The Omega Man, Dragon Ball Z Dead Zone, Lightning in a Bottle, The Wizard of Oz, Goodfellas\\n(J) Lightning in a Bottle, Run Lola Run, Being John Malkovich, The Big Lebowski, The Wizard of Oz\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}