{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"604c5da9-f758-5bf8-bf54-5763b8ad2865","task_key":"bbeh~5fmovie~5frecommendation--full--604c5da9-f758-5bf8-bf54-5763b8ad2865","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Star Wars Episode VI - Return of the Jedi, Back to the Future, Sleeping with the Enemy, Forrest Gump, Independence Day\\n(B) Independence Day, Sleeping with the Enemy, Out of the Past, Ghost in the Shell Arise - Border 2 Ghost Whispers, Back to the Future\\n(C) Star Wars Episode VI - Return of the Jedi, Out of the Past, Independence Day, Back to the Future, Ghost in the Shell Arise - Border 2 Ghost Whispers\\n(D) Forrest Gump, Independence Day, Star Wars Episode VI - Return of the Jedi, Sleeping with the Enemy, Ghost in the Shell Arise - Border 2 Ghost Whispers\\n(E) The Matrix, Forrest Gump, Independence Day, Star Wars Episode VI - Return of the Jedi, Back to the Future\\n(F) Star Wars Episode VI - Return of the Jedi, Ghost in the Shell Arise - Border 2 Ghost Whispers, Out of the Past, Back to the Future, Sleeping with the Enemy\\n(G) Sleeping with the Enemy, Ghost in the Shell Arise - Border 2 Ghost Whispers, Independence Day, Out of the Past, Forrest Gump\\n(H) The Matrix, Forrest Gump, Independence Day, Out of the Past, Back to the Future\\n(I) Ghost in the Shell Arise - Border 2 Ghost Whispers, Out of the Past, Sleeping with the Enemy, Independence Day, Star Wars Episode VI - Return of the Jedi\\n(J) Forrest Gump, Star Wars Episode VI - Return of the Jedi, Independence Day, Ghost in the Shell Arise - Border 2 Ghost Whispers, Out of the Past\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}