{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d068e07f-47c7-59a0-9263-58852ce3fbce","task_key":"bbeh~5fmovie~5frecommendation--full--d068e07f-47c7-59a0-9263-58852ce3fbce","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Babe, Pretty Woman, Game of Death, What We Do in the Shadows, Eastern Promises\\n(B) Pretty Woman, Batman, What We Do in the Shadows, The Mask, Eastern Promises\\n(C) Game of Death, The Mask, Batman, Eastern Promises, The Fugitive\\n(D) Batman, The Fugitive, Eastern Promises, Pretty Woman, Babe\\n(E) Pretty Woman, The Fugitive, Game of Death, Eastern Promises, What We Do in the Shadows\\n(F) What We Do in the Shadows, Pretty Woman, Batman, Eastern Promises, The Fugitive\\n(G) Pretty Woman, Babe, The Fugitive, Batman, The Mask\\n(H) Batman, Pretty Woman, Game of Death, What We Do in the Shadows, The Fugitive\\n(I) Eastern Promises, The Fugitive, Pretty Woman, What We Do in the Shadows, The Mask\\n(J) Eastern Promises, Game of Death, Pretty Woman, Batman, The Mask\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}