{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ea2fdbe5-40c3-5451-8911-8de2bc9a1ea1","task_key":"bbeh~5flinguini--full--ea2fdbe5-40c3-5451-8911-8de2bc9a1ea1","task_revision_id":"1","upstream_id":"","short_description":"Below you see romanised sentences in the Lakhota language and their English…","config":"bbeh_linguini","split":"full","body":"{\"input\":\"Below you see romanised sentences in the Lakhota language and their English translations:\\n\\n1. lakhota ki wičhakte | The Indian killed them.\\n2. matho ki wakte | I killed the bear.\\n3. lakhota ki mačho | The Indian called me.\\n4. tuwa ničho  |  Who called you?\\n5. wičhaša ki tuwa kte | The person killed someone.\\n6. tuwa hi he | Who came?\\n7. matho ki wičhačho | He called the bears.\\n8. yahi čha hi | You came, and he came.\\n9. matho ki hipi ną lakhota ki čhopi | The bears came and called the Indian.\\n10. yahi čha hokšila ki nikte | You came, and the boy killed you.\\n11. lakhota ki wačho ną hokšila ki wakte | I called the Indian and killed the boy.\\n12. hokšila ki wakte čha tuwa lakhota ki wičhačho | I killed the boy, and someone called the Indians.\\n13. lakhota ki hipi čha mayačho | The Indians came, and you called me.\\n\\nTranslate into Lakhota:\\nWhom did I call?\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}