{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"87948111-510a-5062-bab6-2800980a2d21","task_key":"bbeh~5fmovie~5frecommendation--full--87948111-510a-5062-bab6-2800980a2d21","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Pretty Woman, Terminator 2 Judgment Day, The Lion King, The Old Man and the Sea, Batman\\n(B) The Mask, 'night Mother, The Old Man and the Sea, Drag Me to Hell, Terminator 2 Judgment Day\\n(C) The Old Man and the Sea, The Mask, Pretty Woman, Terminator 2 Judgment Day, 'night Mother\\n(D) The Lion King, Pretty Woman, The Old Man and the Sea, Terminator 2 Judgment Day, The Mask\\n(E) The Lion King, Batman, Terminator 2 Judgment Day, Pretty Woman, The Mask\\n(F) Pretty Woman, The Old Man and the Sea, Terminator 2 Judgment Day, The Mask, Drag Me to Hell\\n(G) Drag Me to Hell, The Lion King, Pretty Woman, Terminator 2 Judgment Day, The Old Man and the Sea\\n(H) Batman, The Mask, Pretty Woman, The Old Man and the Sea, Terminator 2 Judgment Day\\n(I) Batman, The Old Man and the Sea, The Mask, The Lion King, Pretty Woman\\n(J) The Old Man and the Sea, Pretty Woman, Batman, 'night Mother, Drag Me to Hell\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}