{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"c6aea4e5-620c-5676-8f19-aafdc921f7ec","task_key":"bbeh~5fnycc--full--c6aea4e5-620c-5676-8f19-aafdc921f7ec","task_revision_id":"1","upstream_id":"","short_description":"The following is a description of a funny cartoon for the New Yorker Caption…","config":"bbeh_nycc","split":"full","body":"{\"input\":\"The following is a description of a funny cartoon for the New Yorker Caption Contest\\n\\n  Description: Two sharks are swimming underwater. One shark has a periscope and a pair of binoculars in its mouth.\\n\\n  Which of the following captions is the funniest?\\n(A) Do I have something in my teeth?\\n(B) I know, I'm just chewing on this until I break the habit.\\n(C) Dummy.\\n(D) It's my new high-fiber diet.\\n(E) \\\"Yeah, I'm a mannequin-eating shark. Gotta problem with that?\\n(F) How's the vegan diet going Carl?\\n(G) I call it 'A Farewell to Arms'.\\n(H) Working on your form?\\n(I) You dumped me to be with that dummy\\n(J) I'm on a cleanse, Sharon.\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}