{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"62f85190-6851-5120-828f-1eda359b7582","task_key":"bbeh~5fmovie~5frecommendation--full--62f85190-6851-5120-828f-1eda359b7582","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) A Special Day, Pulp Fiction, Forrest Gump, Happy Go Lovely, The Fugitive\\n(B) Happy Go Lovely, Pulp Fiction, Dances with Wolves, The Fugitive, David Cross Making America Great Again\\n(C) Forrest Gump, Happy Go Lovely, The Shawshank Redemption, A Special Day, David Cross Making America Great Again\\n(D) A Special Day, The Fugitive, Happy Go Lovely, David Cross Making America Great Again, The Shawshank Redemption\\n(E) Pulp Fiction, Happy Go Lovely, Dances with Wolves, The Fugitive, Forrest Gump\\n(F) Happy Go Lovely, The Fugitive, A Special Day, Dances with Wolves, David Cross Making America Great Again\\n(G) The Fugitive, Dances with Wolves, Pulp Fiction, Forrest Gump, The Shawshank Redemption\\n(H) Happy Go Lovely, David Cross Making America Great Again, The Fugitive, A Special Day, Forrest Gump\\n(I) Dances with Wolves, Happy Go Lovely, The Fugitive, Pulp Fiction, The Shawshank Redemption\\n(J) Forrest Gump, Pulp Fiction, Dances with Wolves, A Special Day, David Cross Making America Great Again\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}