{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a4ee5680-25bd-53d2-b2ac-50ff55555ca8","task_key":"bbeh~5fmovie~5frecommendation--full--a4ee5680-25bd-53d2-b2ac-50ff55555ca8","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) The Matrix, Pulp Fiction, The Spy Next Door, What Love Is, Little Miss Marker\\n(B) The Godfather, The Spy Next Door, The Matrix, Pi, Pulp Fiction\\n(C) Pulp Fiction, The Godfather, What Love Is, The Spy Next Door, The Usual Suspects\\n(D) The Usual Suspects, The Matrix, Pi, The Godfather, Pulp Fiction\\n(E) The Spy Next Door, Pi, What Love Is, Little Miss Marker, The Godfather\\n(F) The Usual Suspects, The Matrix, The Godfather, The Spy Next Door, Little Miss Marker\\n(G) Pulp Fiction, Pi, The Matrix, The Godfather, What Love Is\\n(H) Little Miss Marker, The Spy Next Door, The Usual Suspects, What Love Is, The Godfather\\n(I) The Usual Suspects, Little Miss Marker, The Godfather, The Matrix, What Love Is\\n(J) What Love Is, The Spy Next Door, Little Miss Marker, Pulp Fiction, The Godfather\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}