{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"563de48e-2979-5f40-b2a0-f241ead9e42b","task_key":"bbeh~5fnycc--full--563de48e-2979-5f40-b2a0-f241ead9e42b","task_revision_id":"1","upstream_id":"","short_description":"The following is a description of a funny cartoon for the New Yorker Caption…","config":"bbeh_nycc","split":"full","body":"{\"input\":\"The following is a description of a funny cartoon for the New Yorker Caption Contest\\n\\n  Description: A couple is looking at a treehouse that is for sale. A real estate agent is showing them the property.\\n\\n  Which of the following captions is the funniest?\\n(A) The previous couple had a falling out.\\n(B) The seller is very motivated—he’s too tall to fit in the door now.\\n(C) You only need the secret password to get in\\n(D) This studio has incredible natural light with a rustic feel and is only going for $3500 a month.\\n(E) No flood insurance nessary.\\n(F) ....and this is our in- law suite.\\n(G) You appear to have misunderstood my firm's ad for an open house.\\n(H) Perfectly suitable for two, with room to branch out!\\n(I) Instead of a tree house, think of it as a house in a tree.\\n(J) If you act now we’ll throw in a door!\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}