{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4439fb56-4366-5f98-86ff-c183fa8c7f92","task_key":"bbeh~5fmovie~5frecommendation--full--4439fb56-4366-5f98-86ff-c183fa8c7f92","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Apollo 13, Schindler's List, Pretty Woman, Bad Taste, Dances with Wolves\\n(B) The Santa Clause 2, Braveheart, Dances with Wolves, Bad Taste, My Big Fat Greek Wedding 2\\n(C) Braveheart, Bad Taste, Schindler's List, The Santa Clause 2, Apollo 13\\n(D) My Big Fat Greek Wedding 2, Braveheart, Pretty Woman, Dances with Wolves, Apollo 13\\n(E) Schindler's List, Apollo 13, My Big Fat Greek Wedding 2, Dances with Wolves, Braveheart\\n(F) Apollo 13, The Santa Clause 2, Schindler's List, My Big Fat Greek Wedding 2, Braveheart\\n(G) Dances with Wolves, Braveheart, Pretty Woman, Apollo 13, Bad Taste\\n(H) Pretty Woman, Dances with Wolves, Apollo 13, Braveheart, Schindler's List\\n(I) My Big Fat Greek Wedding 2, The Santa Clause 2, Dances with Wolves, Bad Taste, Pretty Woman\\n(J) Schindler's List, Apollo 13, Bad Taste, My Big Fat Greek Wedding 2, Pretty Woman\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}