{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6f9a3275-a288-5393-b0c5-d95ab6d1e152","task_key":"bbeh~5flinguini--full--6f9a3275-a288-5393-b0c5-d95ab6d1e152","task_revision_id":"1","upstream_id":"","short_description":"Given are Drehu numerals in alphabetical order and their values in ascending…","config":"bbeh_linguini","split":"full","body":"{\"input\":\"Given are Drehu numerals in alphabetical order and their values in ascending order:\\n\\ncaatr nge caako, caatr nge caangömen, caatr nge caaqaihano, ekaatr nge ekengömen, köniatr nge köniko, köniatr nge könipi, köniatr nge köniqaihano, lueatr nge lue, lueatr nge luako, lueatr nge luepi\\n\\n26, 31, 36, 42, 50, 52, 73, 75, 78, 89\\n\\nWrite out in Drehu: 48.\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}