{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e756e1ba-bab8-542f-ada9-b27fd8e5fdb3","task_key":"bbeh~5fmovie~5frecommendation--full--e756e1ba-bab8-542f-ada9-b27fd8e5fdb3","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Raiders of the Lost Ark, House Party 3, Star Wars Episode IV - A New Hope, The Big Lebowski, The Silence of the Lambs\\n(B) Goodfellas, The Big Lebowski, Doctor Dolittle, Pi, Raiders of the Lost Ark\\n(C) Raiders of the Lost Ark, Pi, The Silence of the Lambs, The Big Lebowski, Goodfellas\\n(D) The Silence of the Lambs, Pi, Goodfellas, House Party 3, The Big Lebowski\\n(E) House Party 3, Goodfellas, Pi, The Big Lebowski, Doctor Dolittle\\n(F) The Silence of the Lambs, House Party 3, Raiders of the Lost Ark, Pi, Doctor Dolittle\\n(G) The Big Lebowski, Raiders of the Lost Ark, The Silence of the Lambs, Goodfellas, Star Wars Episode IV - A New Hope\\n(H) The Big Lebowski, Doctor Dolittle, Raiders of the Lost Ark, Goodfellas, Star Wars Episode IV - A New Hope\\n(I) The Silence of the Lambs, Goodfellas, House Party 3, Pi, Doctor Dolittle\\n(J) Pi, The Big Lebowski, The Silence of the Lambs, Doctor Dolittle, Goodfellas\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}