{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4fabe0cc-8fd4-54f8-bfab-147c3b783e53","task_key":"bbeh~5fmovie~5frecommendation--full--4fabe0cc-8fd4-54f8-bfab-147c3b783e53","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Usual Suspects, Braveheart, Crimson Tide, The Fugitive, Too Many Cooks\\n(B) Too Many Cooks, The Usual Suspects, Braveheart, Bandidas, Bad Santa\\n(C) The Usual Suspects, Too Many Cooks, Crimson Tide, Dances with Wolves, Bad Santa\\n(D) The Fugitive, The Usual Suspects, Crimson Tide, Dances with Wolves, Braveheart\\n(E) The Usual Suspects, The Fugitive, Too Many Cooks, Dances with Wolves, Bad Santa\\n(F) The Usual Suspects, Bad Santa, The Fugitive, Bandidas, Dances with Wolves\\n(G) Crimson Tide, Bad Santa, Dances with Wolves, Bandidas, Too Many Cooks\\n(H) Bandidas, Too Many Cooks, The Fugitive, Dances with Wolves, Crimson Tide\\n(I) Bad Santa, Bandidas, Crimson Tide, Too Many Cooks, The Usual Suspects\\n(J) The Usual Suspects, Bad Santa, Crimson Tide, Braveheart, Dances with Wolves\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}