{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"fde5d330-bd25-5a71-b301-9e77e8a1015e","task_key":"bbeh~5fmovie~5frecommendation--full--fde5d330-bd25-5a71-b301-9e77e8a1015e","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Dogfight, The Silence of the Lambs, Forrest Gump, Mission Impossible, Twin Peaks Fire Walk with Me\\n(B) Twin Peaks Fire Walk with Me, The Silence of the Lambs, Forrest Gump, Dogfight, Jurassic Park\\n(C) Dogfight, Joe Somebody, Mission Impossible, Twin Peaks Fire Walk with Me, The Silence of the Lambs\\n(D) Independence Day, Forrest Gump, Joe Somebody, Twin Peaks Fire Walk with Me, Dogfight\\n(E) Mission Impossible, Independence Day, Jurassic Park, Forrest Gump, The Silence of the Lambs\\n(F) Jurassic Park, Mission Impossible, Independence Day, Dogfight, The Silence of the Lambs\\n(G) Independence Day, Jurassic Park, Dogfight, Mission Impossible, Forrest Gump\\n(H) Jurassic Park, Twin Peaks Fire Walk with Me, Forrest Gump, Mission Impossible, Independence Day\\n(I) Jurassic Park, The Silence of the Lambs, Forrest Gump, Joe Somebody, Dogfight\\n(J) Forrest Gump, Independence Day, Joe Somebody, Jurassic Park, Mission Impossible\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}