{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c19483e5-a854-5f27-85c1-7c68377d048a","task_key":"bbeh~5fmovie~5frecommendation--full--c19483e5-a854-5f27-85c1-7c68377d048a","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Forrest Gump, Max Keeble's Big Move, Jurassic Park, Banana Joe, Shine\\n(B) Banana Joe, Die Hard With a Vengeance, Independence Day, Forrest Gump, Jurassic Park\\n(C) Shine, Jurassic Park, Die Hard With a Vengeance, Forrest Gump, Max Keeble's Big Move\\n(D) Max Keeble's Big Move, The Silence of the Lambs, Independence Day, Jurassic Park, Forrest Gump\\n(E) Banana Joe, Max Keeble's Big Move, The Silence of the Lambs, Jurassic Park, Die Hard With a Vengeance\\n(F) Independence Day, Jurassic Park, Banana Joe, Die Hard With a Vengeance, The Silence of the Lambs\\n(G) Independence Day, Jurassic Park, Die Hard With a Vengeance, Forrest Gump, The Silence of the Lambs\\n(H) Shine, The Silence of the Lambs, Forrest Gump, Max Keeble's Big Move, Banana Joe\\n(I) Max Keeble's Big Move, Die Hard With a Vengeance, Shine, Forrest Gump, Banana Joe\\n(J) Die Hard With a Vengeance, Jurassic Park, Banana Joe, Independence Day, Shine\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}