{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"8d82132b-f364-5e33-afb3-4d8ae96f8ddc","task_key":"bbeh~5fspatial~5freasoning--full--8d82132b-f364-5e33-afb3-4d8ae96f8ddc","task_revision_id":"1","upstream_id":"","short_description":"You have been given a pointy-topped regular hexagonal tile map consisting of 2…","config":"bbeh_spatial_reasoning","split":"full","body":"{\"input\":\"You have been given a pointy-topped regular hexagonal tile map consisting of 2 rows, where the first row has one tile and the second row has two tiles. Starting from a vertex, you will move along the edges of these tiles. Initially, you are positioned at the bottom left corner of the map, where you find a Polaroid camera. You move down-right by one step, where you find a hartebeest. You move up-right by one step, where you find an oil filter. You move up by one step, where you find a hot dog. You move up-right by one step, where you find a limousine. You move down-right by one step, where you find a spotted salamander. You move down by one step, where you find a goldfinch. You move down-left by one step, where you find an ice cream. You move up-left by one step. What will you find? Your final answer must be only the object name (e.g., laptop). If you have not previously seen the object at the vertex, your final answer must be \\\"unknown\\\".\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}