{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b889d13c-7faa-5542-ad88-d5235b75cb1e","task_key":"bbeh~5fmovie~5frecommendation--full--b889d13c-7faa-5542-ad88-d5235b75cb1e","task_revision_id":"1","upstream_id":"","short_description":"Which option has more similar movies in terms of whether a group of people will…","config":"bbeh_movie_recommendation","split":"full","body":"{\"input\":\"Which option has more similar movies in terms of whether a group of people will like them or not?\\nOptions:\\n(A) Dead Man Walking, Deuce Bigalow Male Gigolo, The Shawshank Redemption, The Fugitive, Terminator 2 Judgment Day\\n(B) Deuce Bigalow Male Gigolo, Music and Lyrics, The Shawshank Redemption, Braveheart, Dead Man Walking\\n(C) Louis CK Oh My God, Braveheart, The Shawshank Redemption, Music and Lyrics, Dead Man Walking\\n(D) Terminator 2 Judgment Day, Dead Man Walking, Music and Lyrics, Braveheart, The Fugitive\\n(E) Louis CK Oh My God, Deuce Bigalow Male Gigolo, Dead Man Walking, Braveheart, The Fugitive\\n(F) The Fugitive, Music and Lyrics, Deuce Bigalow Male Gigolo, Louis CK Oh My God, Dead Man Walking\\n(G) Louis CK Oh My God, The Fugitive, Braveheart, The Shawshank Redemption, Deuce Bigalow Male Gigolo\\n(H) Louis CK Oh My God, The Shawshank Redemption, Braveheart, Terminator 2 Judgment Day, Dead Man Walking\\n(I) Braveheart, Music and Lyrics, Terminator 2 Judgment Day, The Shawshank Redemption, Louis CK Oh My God\\n(J) The Fugitive, Braveheart, Dead Man Walking, The Shawshank Redemption, Terminator 2 Judgment Day\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}