{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6922e71b-58b8-5d71-8517-759a1856a9e7","task_key":"bbeh~5fmovie~5frecommendation--full--6922e71b-58b8-5d71-8517-759a1856a9e7","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Schindler's List, Primary Colors, Interstate 60, The Fugitive, Careful What You Wish For\\n(B) The Shawshank Redemption, Schindler's List, The Mask, Interstate 60, The Fugitive\\n(C) The Fugitive, Primary Colors, Careful What You Wish For, Braveheart, Interstate 60\\n(D) Primary Colors, The Mask, Braveheart, Interstate 60, The Shawshank Redemption\\n(E) Careful What You Wish For, Primary Colors, The Mask, Schindler's List, Interstate 60\\n(F) Braveheart, The Fugitive, The Shawshank Redemption, The Mask, Interstate 60\\n(G) The Fugitive, Braveheart, Schindler's List, The Mask, The Shawshank Redemption\\n(H) Braveheart, The Mask, The Shawshank Redemption, Schindler's List, Interstate 60\\n(I) Braveheart, Careful What You Wish For, The Fugitive, The Shawshank Redemption, Schindler's List\\n(J) Primary Colors, The Mask, The Shawshank Redemption, The Fugitive, Braveheart\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}