{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"d97a769f-b026-5691-9c25-c321199d1c63","task_key":"bbeh~5fmovie~5frecommendation--full--d97a769f-b026-5691-9c25-c321199d1c63","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Get Smart, Music Box, College, Raiders of the Lost Ark, Men in Black\\n(B) The Fugitive, The Silence of the Lambs, College, Raiders of the Lost Ark, Get Smart\\n(C) Music Box, Get Smart, College, Raiders of the Lost Ark, Fargo\\n(D) The Silence of the Lambs, Men in Black, Raiders of the Lost Ark, Music Box, College\\n(E) Raiders of the Lost Ark, Get Smart, The Silence of the Lambs, The Fugitive, Men in Black\\n(F) Fargo, The Fugitive, The Silence of the Lambs, Raiders of the Lost Ark, Men in Black\\n(G) Get Smart, Music Box, Fargo, College, The Fugitive\\n(H) Men in Black, Music Box, Raiders of the Lost Ark, Get Smart, The Fugitive\\n(I) Raiders of the Lost Ark, Get Smart, College, Men in Black, Fargo\\n(J) College, Raiders of the Lost Ark, Fargo, The Silence of the Lambs, Men in Black\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}