{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d57fef47-6533-5b6e-b679-a1d9007b1136","task_key":"bbeh~5fmovie~5frecommendation--full--d57fef47-6533-5b6e-b679-a1d9007b1136","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Bridget Jones's Diary, The Double Life of Veronique, Terminator 2 Judgment Day, The Lion King, Schindler's List\\n(B) The Double Life of Veronique, Aladdin, The Lion King, Schindler's List, Terminator 2 Judgment Day\\n(C) Bridget Jones's Diary, The Shawshank Redemption, Schindler's List, Aladdin, Terminator 2 Judgment Day\\n(D) The Shawshank Redemption, Bridget Jones's Diary, Schindler's List, Terminator 2 Judgment Day, The Lion King\\n(E) The Double Life of Veronique, Schindler's List, The Lion King, Cujo, The Shawshank Redemption\\n(F) The Shawshank Redemption, Bridget Jones's Diary, The Lion King, Terminator 2 Judgment Day, Aladdin\\n(G) Aladdin, Bridget Jones's Diary, The Lion King, The Double Life of Veronique, Cujo\\n(H) Bridget Jones's Diary, The Lion King, The Double Life of Veronique, The Shawshank Redemption, Schindler's List\\n(I) Aladdin, Cujo, Terminator 2 Judgment Day, The Lion King, Bridget Jones's Diary\\n(J) Terminator 2 Judgment Day, The Shawshank Redemption, The Lion King, Aladdin, Schindler's List\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}