{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6eb5f942-ae6e-5572-ac25-219955795113","task_key":"bbeh~5fmovie~5frecommendation--full--6eb5f942-ae6e-5572-ac25-219955795113","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Independence Day, Pulp Fiction, Forrest Gump, Raiders of the Lost Ark, Inception\\n(B) Who's Afraid of Virginia Woolf, Inception, Raiders of the Lost Ark, Fantastic Mr Fox, Pulp Fiction\\n(C) Raiders of the Lost Ark, Inception, Fantastic Mr Fox, Clash of the Titans, Who's Afraid of Virginia Woolf\\n(D) Forrest Gump, Who's Afraid of Virginia Woolf, Independence Day, Pulp Fiction, Inception\\n(E) Fantastic Mr Fox, Independence Day, Raiders of the Lost Ark, Clash of the Titans, Pulp Fiction\\n(F) Inception, Raiders of the Lost Ark, Fantastic Mr Fox, Forrest Gump, Independence Day\\n(G) Pulp Fiction, Inception, Fantastic Mr Fox, Who's Afraid of Virginia Woolf, Independence Day\\n(H) Independence Day, Who's Afraid of Virginia Woolf, Clash of the Titans, Forrest Gump, Raiders of the Lost Ark\\n(I) Inception, Raiders of the Lost Ark, Who's Afraid of Virginia Woolf, Clash of the Titans, Pulp Fiction\\n(J) Who's Afraid of Virginia Woolf, Pulp Fiction, Fantastic Mr Fox, Raiders of the Lost Ark, Clash of the Titans\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}