{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d1052df3-29de-5221-9c70-bee8dc0a70c3","task_key":"bbeh~5fmovie~5frecommendation--full--d1052df3-29de-5221-9c70-bee8dc0a70c3","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Indiana Jones and the Temple of Doom, The Fugitive, Fargo, Independence Day, The Silence of the Lambs\\n(B) Care Bears Movie II A New Generation, The Fugitive, Independence Day, Fargo, The Silence of the Lambs\\n(C) Care Bears Movie II A New Generation, Independence Day, Fargo, The Silence of the Lambs, Lights Out\\n(D) Care Bears Movie II A New Generation, Indiana Jones and the Temple of Doom, Lights Out, Independence Day, Noises Off\\n(E) Independence Day, Indiana Jones and the Temple of Doom, Fargo, Care Bears Movie II A New Generation, The Fugitive\\n(F) The Fugitive, Indiana Jones and the Temple of Doom, Fargo, Lights Out, Noises Off\\n(G) Lights Out, The Fugitive, The Silence of the Lambs, Independence Day, Care Bears Movie II A New Generation\\n(H) Care Bears Movie II A New Generation, Lights Out, Indiana Jones and the Temple of Doom, Fargo, Noises Off\\n(I) Fargo, Independence Day, Care Bears Movie II A New Generation, Noises Off, Lights Out\\n(J) Lights Out, Care Bears Movie II A New Generation, The Fugitive, Indiana Jones and the Temple of Doom, Fargo\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}