{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b4caefa9-77d0-50cf-b66d-7813fd4e07d1","task_key":"bbeh~5fmovie~5frecommendation--full--b4caefa9-77d0-50cf-b66d-7813fd4e07d1","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Fugitive, The Care Bears Movie, Dances with Wolves, Pulp Fiction, Forrest Gump\\n(B) Pulp Fiction, Baby Secret of the Lost Legend, The Fiendish Plot of Dr Fu Manchu, The Care Bears Movie, The Shawshank Redemption\\n(C) The Shawshank Redemption, Forrest Gump, The Fiendish Plot of Dr Fu Manchu, The Fugitive, Dances with Wolves\\n(D) Pulp Fiction, Dances with Wolves, The Fugitive, The Shawshank Redemption, Forrest Gump\\n(E) The Fiendish Plot of Dr Fu Manchu, Pulp Fiction, Dances with Wolves, The Fugitive, The Care Bears Movie\\n(F) Baby Secret of the Lost Legend, The Shawshank Redemption, The Care Bears Movie, Forrest Gump, The Fugitive\\n(G) The Fugitive, Forrest Gump, Dances with Wolves, The Fiendish Plot of Dr Fu Manchu, The Care Bears Movie\\n(H) The Shawshank Redemption, Baby Secret of the Lost Legend, The Care Bears Movie, Pulp Fiction, The Fugitive\\n(I) The Shawshank Redemption, The Fiendish Plot of Dr Fu Manchu, The Fugitive, Baby Secret of the Lost Legend, Dances with Wolves\\n(J) Baby Secret of the Lost Legend, The Care Bears Movie, The Fugitive, Forrest Gump, The Fiendish Plot of Dr Fu Manchu\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}