{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a23f48e0-b80e-536d-8fba-92e58e596548","task_key":"bbeh~5flinguini--full--a23f48e0-b80e-536d-8fba-92e58e596548","task_revision_id":"1","upstream_id":"","short_description":"Here are some sentences in Basque as well as their English translations in…","config":"bbeh_linguini","split":"full","body":"{\"input\":\"Here are some sentences in Basque as well as their English translations in arbitrary order. One of the English sentences corresponds to two sentences in Basque:\\n\\n1. ahaztu ditut\\n2. ahaztu zaizkit\\n3. ahaztu zaizu\\n4. hurbildu natzaizue\\n5. hurbildu zait\\n6. lagundu ditugu\\n7. lagundu dituzu\\n8. lagundu dute\\n9. lagundu nauzue\\n10. mintzatu natzaizu\\n11. mintzatu gatzaizkizue\\n12. mintzatu zaizkigu\\n13. ukitu ditugu\\n14. ukitu naut\\n\\nA. you(sg) forgot him\\nB. they talked to us\\nC. I approached you(pl)\\nD. I talked to you(sg)\\nE. we helped them\\nF. you(pl) helped me\\nG. he approached me\\nH. we touched them\\nI. they touched me\\nJ. you(sg) helped them\\nK. they helped him\\nL. we talked to you(pl)\\nM. I forgot them\\n\\nTranslate into English:\\nhurbildu gatzaizkizu\\n\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}