{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"4e4e2e40-6888-5bb5-a83f-8e359b34eee0","task_key":"bbeh~5fspatial~5freasoning--full--4e4e2e40-6888-5bb5-a83f-8e359b34eee0","task_revision_id":"1","upstream_id":"","short_description":"You have been given a 3 by 3 pointy-topped regular rhombus tile map. Starting…","config":"bbeh_spatial_reasoning","split":"full","body":"{\"input\":\"You have been given a 3 by 3 pointy-topped regular rhombus tile map. Starting from a vertex, you will move along the edges of the grid. Initially, you are positioned at the right corner of the grid, where you find an acorn. You move down-left by one step, where you find a toilet paper. You move up-left by one step, where you find an analog clock. You move down-left by one step, where you find a library. You move up-left by one step, where you find a grocery store. You move up-right by one step, where you find a water jug. You move up-right by one step, where you find a station wagon. You move down-right by one step, where you find a half-track. You move down-left by one step. What will you find? Your final answer must be only the object name (e.g., laptop). If you have not previously seen the object at the vertex, your final answer must be \\\"unknown\\\".\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}