{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"536baa00-a52d-5a66-9087-d084e514e1ca","task_key":"bbeh~5fmovie~5frecommendation--full--536baa00-a52d-5a66-9087-d084e514e1ca","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Mortal Kombat The Journey Begins, Léon The Professional, Pulp Fiction, Kill Bill Vol 2, The Godfather\\n(B) Léon The Professional, Kill Bill Vol 2, The Godfather, Pulp Fiction, Requiem for a Dream\\n(C) Pulp Fiction, The Godfather, Sin Nombre, Pushing Hands, Léon The Professional\\n(D) Kill Bill Vol 2, The Godfather, Sin Nombre, Mortal Kombat The Journey Begins, Pushing Hands\\n(E) Sin Nombre, Mortal Kombat The Journey Begins, Kill Bill Vol 2, Léon The Professional, The Godfather\\n(F) Pulp Fiction, Pushing Hands, Sin Nombre, Léon The Professional, Requiem for a Dream\\n(G) Sin Nombre, Pushing Hands, Mortal Kombat The Journey Begins, Pulp Fiction, The Godfather\\n(H) Pushing Hands, Mortal Kombat The Journey Begins, The Godfather, Sin Nombre, Requiem for a Dream\\n(I) Mortal Kombat The Journey Begins, Requiem for a Dream, Kill Bill Vol 2, Léon The Professional, Sin Nombre\\n(J) Pushing Hands, The Godfather, Mortal Kombat The Journey Begins, Pulp Fiction, Kill Bill Vol 2\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}