{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e93c1f5d-47c0-59e4-83b2-c515ebefb60c","task_key":"bbeh~5fmovie~5frecommendation--full--e93c1f5d-47c0-59e4-83b2-c515ebefb60c","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Killer Movie, Stealth, True Lies, Forrest Gump, The Last Man on Earth\\n(B) Stealth, Killer Movie, Jurassic Park, True Lies, Die Hard With a Vengeance\\n(C) Killer Movie, Stealth, The Last Man on Earth, Forrest Gump, Die Hard With a Vengeance\\n(D) Killer Movie, Apollo 13, The Last Man on Earth, Stealth, Forrest Gump\\n(E) True Lies, Forrest Gump, Jurassic Park, Die Hard With a Vengeance, Apollo 13\\n(F) Jurassic Park, The Last Man on Earth, Killer Movie, True Lies, Stealth\\n(G) The Last Man on Earth, True Lies, Stealth, Killer Movie, Apollo 13\\n(H) Apollo 13, Die Hard With a Vengeance, Jurassic Park, Forrest Gump, Stealth\\n(I) Apollo 13, Jurassic Park, Forrest Gump, Killer Movie, True Lies\\n(J) The Last Man on Earth, Killer Movie, Die Hard With a Vengeance, Stealth, Apollo 13\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}