{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"84e24635-c37a-5c98-b09a-8ec2aa00a328","task_key":"bbeh~5fmovie~5frecommendation--full--84e24635-c37a-5c98-b09a-8ec2aa00a328","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Secretary, Independence Day, The Lifeguard, The Human Condition III, Forrest Gump\\n(B) Secretary, Forrest Gump, Star Wars Episode IV - A New Hope, The Lifeguard, Independence Day\\n(C) Secretary, The Lifeguard, The Human Condition III, Star Wars Episode IV - A New Hope, Independence Day\\n(D) Star Wars Episode IV - A New Hope, Forrest Gump, The Lifeguard, The Terminator, Independence Day\\n(E) The Human Condition III, Independence Day, Secretary, The Terminator, Forrest Gump\\n(F) The Terminator, The Human Condition III, Secretary, The Lifeguard, Independence Day\\n(G) Independence Day, The Terminator, Star Wars Episode VI - Return of the Jedi, Forrest Gump, Star Wars Episode IV - A New Hope\\n(H) Independence Day, Star Wars Episode IV - A New Hope, The Terminator, The Lifeguard, Star Wars Episode VI - Return of the Jedi\\n(I) Independence Day, Forrest Gump, The Lifeguard, Star Wars Episode VI - Return of the Jedi, The Terminator\\n(J) Independence Day, Star Wars Episode VI - Return of the Jedi, Secretary, Forrest Gump, Star Wars Episode IV - A New Hope\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}