{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6c03c048-6044-562e-a73f-728f170f9ef2","task_key":"bbeh~5fmovie~5frecommendation--full--6c03c048-6044-562e-a73f-728f170f9ef2","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Extreme Measures, Jurassic Park, Independence Day, Beauty and the Beast, The Silence of the Lambs\\n(B) Independence Day, The Silence of the Lambs, Jurassic Park, Extreme Measures, Forrest Gump\\n(C) Kicking Off, Independence Day, Jurassic Park, Beauty and the Beast, Forrest Gump\\n(D) Kicking Off, Jurassic Park, Beauty and the Beast, The Bag Man, The Silence of the Lambs\\n(E) Kicking Off, Forrest Gump, Jurassic Park, Beauty and the Beast, The Silence of the Lambs\\n(F) Beauty and the Beast, Extreme Measures, Kicking Off, Independence Day, The Bag Man\\n(G) Independence Day, Forrest Gump, Beauty and the Beast, Jurassic Park, The Silence of the Lambs\\n(H) Kicking Off, Jurassic Park, Extreme Measures, The Bag Man, Beauty and the Beast\\n(I) Forrest Gump, Independence Day, Jurassic Park, The Silence of the Lambs, The Bag Man\\n(J) Beauty and the Beast, Forrest Gump, Independence Day, Jurassic Park, Extreme Measures\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}