{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8efa2f56-4c7c-513a-8044-f10145e05ac8","task_key":"bbeh~5fmovie~5frecommendation--full--8efa2f56-4c7c-513a-8044-f10145e05ac8","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Evolution, Fight Club, Sorority House Massacre, Memento, Kill Bill Vol 1\\n(B) Evolution, Fight Club, Kill Bill Vol 1, Memento, Kill Bill Vol 2\\n(C) The Shawshank Redemption, Evolution, Scooby-Doo Goes Hollywood, Sorority House Massacre, Fight Club\\n(D) Kill Bill Vol 2, Evolution, Scooby-Doo Goes Hollywood, Kill Bill Vol 1, Sorority House Massacre\\n(E) Memento, Fight Club, Kill Bill Vol 1, The Shawshank Redemption, Kill Bill Vol 2\\n(F) Sorority House Massacre, Fight Club, Scooby-Doo Goes Hollywood, Kill Bill Vol 2, Evolution\\n(G) Kill Bill Vol 1, Sorority House Massacre, The Shawshank Redemption, Evolution, Scooby-Doo Goes Hollywood\\n(H) Fight Club, Evolution, Scooby-Doo Goes Hollywood, Sorority House Massacre, Memento\\n(I) Sorority House Massacre, Evolution, Kill Bill Vol 1, Fight Club, Kill Bill Vol 2\\n(J) The Shawshank Redemption, Sorority House Massacre, Memento, Kill Bill Vol 2, Evolution\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}