{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ef904420-167f-5109-80a4-09157eb01c82","task_key":"bbeh~5fmovie~5frecommendation--full--ef904420-167f-5109-80a4-09157eb01c82","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Pulp Fiction, The Shawshank Redemption, Forrest Gump, Dances with Wolves, While You Were Sleeping\\n(B) Women in Trouble, Night at the Museum Secret of the Tomb, While You Were Sleeping, The Fugitive, The Shawshank Redemption\\n(C) Forrest Gump, The Shawshank Redemption, Dances with Wolves, The Fugitive, Women in Trouble\\n(D) While You Were Sleeping, Night at the Museum Secret of the Tomb, The Shawshank Redemption, Women in Trouble, Dances with Wolves\\n(E) Women in Trouble, Night at the Museum Secret of the Tomb, While You Were Sleeping, The Shawshank Redemption, Forrest Gump\\n(F) Women in Trouble, Night at the Museum Secret of the Tomb, Dances with Wolves, The Fugitive, Forrest Gump\\n(G) The Fugitive, Night at the Museum Secret of the Tomb, Forrest Gump, While You Were Sleeping, Pulp Fiction\\n(H) Pulp Fiction, Forrest Gump, Dances with Wolves, The Fugitive, The Shawshank Redemption\\n(I) Pulp Fiction, Dances with Wolves, While You Were Sleeping, The Shawshank Redemption, Night at the Museum Secret of the Tomb\\n(J) While You Were Sleeping, The Shawshank Redemption, Pulp Fiction, Night at the Museum Secret of the Tomb, Women in Trouble\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}