{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8e2c7b69-0b31-551c-a183-32249ffcf32e","task_key":"bbeh~5fmovie~5frecommendation--full--8e2c7b69-0b31-551c-a183-32249ffcf32e","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Gintama The Final Chapter - Be Forever Yorozuya, Los Angeles Plays Itself, Pretty Woman, The Fugitive, Batman\\n(B) The Lion King, The Mask, The Fugitive, Pretty Woman, Gintama The Final Chapter - Be Forever Yorozuya\\n(C) Gintama The Final Chapter - Be Forever Yorozuya, Los Angeles Plays Itself, Pretty Woman, The Mask, Doctor Who Last Christmas\\n(D) Doctor Who Last Christmas, Pretty Woman, The Lion King, Batman, The Fugitive\\n(E) Gintama The Final Chapter - Be Forever Yorozuya, Los Angeles Plays Itself, The Lion King, Batman, The Mask\\n(F) The Lion King, Batman, Pretty Woman, The Mask, The Fugitive\\n(G) The Lion King, The Mask, Doctor Who Last Christmas, Los Angeles Plays Itself, Gintama The Final Chapter - Be Forever Yorozuya\\n(H) The Fugitive, Pretty Woman, Gintama The Final Chapter - Be Forever Yorozuya, Batman, Doctor Who Last Christmas\\n(I) The Lion King, Pretty Woman, Doctor Who Last Christmas, Gintama The Final Chapter - Be Forever Yorozuya, The Fugitive\\n(J) Batman, The Mask, Doctor Who Last Christmas, Los Angeles Plays Itself, The Lion King\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}