{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"e3ff9346-d732-5fe7-980e-b70fbff2de7d","task_key":"bbeh~5fspatial~5freasoning--full--e3ff9346-d732-5fe7-980e-b70fbff2de7d","task_revision_id":"1","upstream_id":"","short_description":"You have been given a pointy-topped regular hexagonal tile map consisting of 2…","config":"bbeh_spatial_reasoning","split":"full","body":"{\"input\":\"You have been given a pointy-topped regular hexagonal tile map consisting of 2 rows, where the first row has one tile and the second row has two tiles. Starting from a vertex, you will move along the edges of these tiles. Initially, you are positioned at the top corner of the map, where you find a jackfruit. You move down-left by one step, where you find a Flat-Coated Retriever. You move down by one step, where you find a school bus. You move down-left by one step, where you find a sea slug. You move down by one step, where you find a wallaby. You move down-right by one step, where you find a French horn. You move up-right by one step, where you find a horse-drawn vehicle. You move up by one step, where you find a garter snake. You move up-left by one step. What will you find? Your final answer must be only the object name (e.g., laptop). If you have not previously seen the object at the vertex, your final answer must be \\\"unknown\\\".\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}