{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7c2b4e20-09d8-5ff5-bfbb-c34e08a01bef","task_key":"bbeh~5fmovie~5frecommendation--full--7c2b4e20-09d8-5ff5-bfbb-c34e08a01bef","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Forrest Gump, Batman, Night of the Living Dead, Revenge for Jolly!, Ghost Ship\\n(B) Revenge for Jolly!, The Silence of the Lambs, American Beauty, Forrest Gump, Batman\\n(C) Ghost Ship, Jurassic Park, Revenge for Jolly!, Forrest Gump, The Silence of the Lambs\\n(D) American Beauty, Jurassic Park, Revenge for Jolly!, Batman, The Silence of the Lambs\\n(E) American Beauty, Jurassic Park, The Silence of the Lambs, Forrest Gump, Batman\\n(F) Jurassic Park, Batman, Revenge for Jolly!, The Silence of the Lambs, Forrest Gump\\n(G) American Beauty, Night of the Living Dead, The Silence of the Lambs, Forrest Gump, Jurassic Park\\n(H) Forrest Gump, The Silence of the Lambs, Batman, Jurassic Park, Ghost Ship\\n(I) Revenge for Jolly!, American Beauty, Jurassic Park, Night of the Living Dead, Batman\\n(J) Revenge for Jolly!, Ghost Ship, The Silence of the Lambs, Batman, Night of the Living Dead\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}