{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1e0eb1e1-2def-565f-90f3-fcc77f61aab7","task_key":"bbeh~5fmovie~5frecommendation--full--1e0eb1e1-2def-565f-90f3-fcc77f61aab7","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) SupermanDoomsday, Pi, X-Men, Police Academy Mission to Moscow, Carnival of Souls\\n(B) X-Men, The Godfather, SupermanDoomsday, Police Academy Mission to Moscow, Carnival of Souls\\n(C) The Matrix, The Godfather, American Beauty, Pi, Police Academy Mission to Moscow\\n(D) The Godfather, American Beauty, The Matrix, X-Men, Pi\\n(E) The Matrix, Police Academy Mission to Moscow, SupermanDoomsday, Carnival of Souls, Pi\\n(F) The Matrix, American Beauty, Pi, Police Academy Mission to Moscow, Carnival of Souls\\n(G) Pi, The Godfather, Carnival of Souls, Police Academy Mission to Moscow, American Beauty\\n(H) Police Academy Mission to Moscow, Pi, Carnival of Souls, X-Men, American Beauty\\n(I) SupermanDoomsday, Police Academy Mission to Moscow, American Beauty, Carnival of Souls, X-Men\\n(J) Police Academy Mission to Moscow, The Godfather, The Matrix, X-Men, American Beauty\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}