{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"285fa500-98ef-5f0f-8328-6eca85e5f7de","task_key":"bbeh~5fweb~5fof~5flies--full--285fa500-98ef-5f0f-8328-6eca85e5f7de","task_revision_id":"1","upstream_id":"","short_description":"In this question, assume each person either always tells the truth or always…","config":"bbeh_web_of_lies","split":"full","body":"{\"input\":\"In this question, assume each person either always tells the truth or always lies. The person at the bookstore says the person at the hotel lies. The person at the library says the person at the park tells the truth. The person at the museum thinks their neighbor is telling the truth. Grace is at the airport. Beatriz is at the school. The person at the beach says the person at the hotel lies. Ryan is at the library. The person at the hotel says the person at the museum lies. Emily is at the museum. The person at the museum says the person at the school tells the truth. The person at the farm tells the truth. Isabella is at the hotel. Nia is at the bookstore. The person at the park says the person at the bookstore tells the truth. Olivia is at the beach. Jaxon is at the park. The person at the hotel says the person at the airport lies. The person at the museum lies. Nadia is at the farm. Does the person at the library tell the truth? Does the person at the park tell the truth? Does the person at the bookstore tell the truth? Your answer should be a list of three words, yes or no (for example, yes, no, yes).\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}