{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1e1a8723-b87f-5866-885f-5933c1b8c24e","task_key":"bbeh~5fmovie~5frecommendation--full--1e1a8723-b87f-5866-885f-5933c1b8c24e","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Shawshank Redemption, Nina's Heavenly Delights, American Pie Presents The Book of Love, Aladdin, Terminator 2 Judgment Day\\n(B) American Pie Presents The Book of Love, The Shawshank Redemption, Terminator 2 Judgment Day, The Lion King, Aladdin\\n(C) The Lion King, The Recruit, Aladdin, Schindler's List, The Shawshank Redemption\\n(D) Aladdin, The Lion King, Schindler's List, American Pie Presents The Book of Love, Terminator 2 Judgment Day\\n(E) American Pie Presents The Book of Love, Schindler's List, Terminator 2 Judgment Day, The Shawshank Redemption, The Recruit\\n(F) American Pie Presents The Book of Love, The Shawshank Redemption, Terminator 2 Judgment Day, The Lion King, Nina's Heavenly Delights\\n(G) Terminator 2 Judgment Day, Nina's Heavenly Delights, The Shawshank Redemption, Aladdin, Schindler's List\\n(H) Terminator 2 Judgment Day, Aladdin, The Lion King, The Shawshank Redemption, Schindler's List\\n(I) The Recruit, Schindler's List, The Lion King, American Pie Presents The Book of Love, Nina's Heavenly Delights\\n(J) Nina's Heavenly Delights, The Recruit, The Lion King, Aladdin, The Shawshank Redemption\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}