{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"0664f10a-2ddf-5585-be3d-2996b8c1d92a","task_key":"bbeh~5fmovie~5frecommendation--full--0664f10a-2ddf-5585-be3d-2996b8c1d92a","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Vicky Cristina Barcelona, Star Wars Episode VI - Return of the Jedi, Back to the Future, Making the Grade, The Boy and the Beast\\n(B) Forrest Gump, Vicky Cristina Barcelona, The Lord of the Rings The Fellowship of the Ring, The Boy and the Beast, Star Wars Episode VI - Return of the Jedi\\n(C) Back to the Future, The Boy and the Beast, The Lord of the Rings The Fellowship of the Ring, The Silence of the Lambs, Forrest Gump\\n(D) Forrest Gump, Back to the Future, Star Wars Episode VI - Return of the Jedi, The Silence of the Lambs, The Lord of the Rings The Fellowship of the Ring\\n(E) Vicky Cristina Barcelona, The Boy and the Beast, Star Wars Episode VI - Return of the Jedi, The Silence of the Lambs, Back to the Future\\n(F) Forrest Gump, Star Wars Episode VI - Return of the Jedi, The Lord of the Rings The Fellowship of the Ring, Vicky Cristina Barcelona, Back to the Future\\n(G) Star Wars Episode VI - Return of the Jedi, Forrest Gump, The Lord of the Rings The Fellowship of the Ring, The Boy and the Beast, The Silence of the Lambs\\n(H) Making the Grade, The Lord of the Rings The Fellowship of the Ring, Star Wars Episode VI - Return of the Jedi, The Silence of the Lambs, The Boy and the Beast\\n(I) Vicky Cristina Barcelona, Making the Grade, The Boy and the Beast, The Silence of the Lambs, Forrest Gump\\n(J) Back to the Future, The Silence of the Lambs, The Boy and the Beast, Star Wars Episode VI - Return of the Jedi, The Lord of the Rings The Fellowship of the Ring\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}