{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c6a3e559-abb6-5270-ab1a-f787aa587095","task_key":"bbeh~5fmovie~5frecommendation--full--c6a3e559-abb6-5270-ab1a-f787aa587095","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Man of the House, The Neon Demon, Schindler's List, Day for Night, Dances with Wolves\\n(B) Day for Night, The Lion King, The Neon Demon, Dances with Wolves, The Shawshank Redemption\\n(C) Schindler's List, The Lion King, The Neon Demon, The Fugitive, Dances with Wolves\\n(D) The Neon Demon, Schindler's List, The Lion King, The Shawshank Redemption, Man of the House\\n(E) Schindler's List, Dances with Wolves, The Fugitive, The Lion King, The Shawshank Redemption\\n(F) Man of the House, Schindler's List, The Neon Demon, The Lion King, Day for Night\\n(G) Dances with Wolves, The Shawshank Redemption, The Neon Demon, Schindler's List, The Lion King\\n(H) Dances with Wolves, Schindler's List, Day for Night, Man of the House, The Fugitive\\n(I) Day for Night, Man of the House, Dances with Wolves, The Lion King, The Fugitive\\n(J) The Shawshank Redemption, Schindler's List, Day for Night, The Lion King, Man of the House\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}