{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8f954874-957c-57c9-910e-25474b5b74a8","task_key":"bbeh~5fmovie~5frecommendation--full--8f954874-957c-57c9-910e-25474b5b74a8","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Professional, A Night to Remember, Kinsey, The Swan Princess, Léon The Professional\\n(B) Léon The Professional, One Flew Over the Cuckoo's Nest, Kinsey, The Professional, Goodfellas\\n(C) One Flew Over the Cuckoo's Nest, The Swan Princess, Rear Window, Léon The Professional, Goodfellas\\n(D) One Flew Over the Cuckoo's Nest, A Night to Remember, The Swan Princess, Goodfellas, Kinsey\\n(E) Rear Window, Léon The Professional, Kinsey, One Flew Over the Cuckoo's Nest, A Night to Remember\\n(F) Kinsey, The Swan Princess, A Night to Remember, Rear Window, The Professional\\n(G) The Professional, Rear Window, Kinsey, The Swan Princess, Goodfellas\\n(H) Léon The Professional, One Flew Over the Cuckoo's Nest, The Professional, Goodfellas, Rear Window\\n(I) One Flew Over the Cuckoo's Nest, Léon The Professional, A Night to Remember, Kinsey, The Professional\\n(J) A Night to Remember, Léon The Professional, The Swan Princess, Rear Window, Kinsey\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}