{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ceb35087-21b1-5d07-ae6c-8d8b9dec29bc","task_key":"bbeh~5fspatial~5freasoning--full--ceb35087-21b1-5d07-ae6c-8d8b9dec29bc","task_revision_id":"1","upstream_id":"","short_description":"You have been given a 3 by 3 pointy-topped regular rhombus tile map. Starting…","config":"bbeh_spatial_reasoning","split":"full","body":"{\"input\":\"You have been given a 3 by 3 pointy-topped regular rhombus tile map. Starting from a vertex, you will move along the edges of the grid. Initially, you are positioned at the left corner of the grid, where you find a sulphur-crested cockatoo. You move down-right by one step, where you find a plow. You move down-right by one step, where you find a Malinois. You move up-right by one step, where you find a barbershop. You move up-left by one step, where you find a washing machine. You move up-left by one step, where you find a mongoose. You move up-right by one step, where you find a sarong. You move down-right by one step, where you find an Afghan Hound. You move down-left by one step. What will you find? Your final answer must be only the object name (e.g., laptop). If you have not previously seen the object at the vertex, your final answer must be \\\"unknown\\\".\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}