{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"52a36cc3-107b-5036-80b1-c3054caf5aa3","task_key":"bbeh~5fweb~5fof~5flies--full--52a36cc3-107b-5036-80b1-c3054caf5aa3","task_revision_id":"1","upstream_id":"","short_description":"In this question, assume each person either always tells the truth or always…","config":"bbeh_web_of_lies","split":"full","body":"{\"input\":\"In this question, assume each person either always tells the truth or always lies. The person at the cafe says the person at the skate park lies. Emily is at the beach. Zara is at the shopping mall. Nadia is at the skate park. The person at the art gallery tells the truth. Max is at the train station. Luna is at the movie theater. Olivia is at the cafe. The person at the cafe says the person at the movie theater lies. The person at the library says the person at the cafe tells the truth. The person at the bookstore tells the truth. Theo is at the library. The person at the bookstore says the person at the vineyard lies. Jake is at the art gallery. The person at the shopping mall lies. The person at the bookstore says the person at the planetarium tells the truth. The person at the cafe says the person at the train station lies. The person at the beach tells the truth. Charlie is at the vineyard. Farid is at the planetarium. The person at the skate park says the person at the bookstore lies. Nia is at the bookstore. Does the person at the library tell the truth? Does the person at the cafe tell the truth? Does the person at the skate park tell the truth? Your answer should be a list of three words, yes or no (for example, yes, no, yes).\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}