{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"07406fb3-aa70-5f1c-8ceb-407272391be8","task_key":"bbeh~5fcausal~5funderstanding--full--07406fb3-aa70-5f1c-8ceb-407272391be8","task_revision_id":"1","upstream_id":"","short_description":"Question: Reagent X is added to a mixture. The mixture becomes hot, otherwise…","config":"bbeh_causal_understanding","split":"full","body":"{\"input\":\"Question: Reagent X is added to a mixture. The mixture becomes hot, otherwise the mixture would have remained cold. In both cases (hot or cold), the mixture would have turned to crystal. The mixture turns to crystal. Is adding Reagent X to the mixture a necessary cause for the crystal formation? Reply based on the answer a logician would give.\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}