{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"996bd0ef-42c8-5606-8836-28115c3be0ab","task_key":"bbeh~5fmovie~5frecommendation--full--996bd0ef-42c8-5606-8836-28115c3be0ab","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Children of the Corn III, Jurassic Park, The Burrowers, Hood of Horror, The Fugitive\\n(B) Braveheart, Hood of Horror, The Burrowers, The Fugitive, Children of the Corn III\\n(C) Forrest Gump, Jurassic Park, Braveheart, The Fugitive, Children of the Corn III\\n(D) Braveheart, The Silence of the Lambs, Hood of Horror, Forrest Gump, The Burrowers\\n(E) Braveheart, Forrest Gump, Hood of Horror, The Fugitive, Jurassic Park\\n(F) The Silence of the Lambs, Forrest Gump, Braveheart, The Fugitive, Jurassic Park\\n(G) Braveheart, The Silence of the Lambs, Children of the Corn III, Hood of Horror, Forrest Gump\\n(H) The Silence of the Lambs, The Burrowers, Jurassic Park, Children of the Corn III, Hood of Horror\\n(I) The Silence of the Lambs, The Burrowers, Forrest Gump, Hood of Horror, Jurassic Park\\n(J) The Burrowers, Hood of Horror, Children of the Corn III, The Silence of the Lambs, Braveheart\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}