{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbeh","formal_name":"BIG-Bench Extra Hard (BBEH)","introduction":"BIG-Bench Extra Hard evaluates language-model reasoning through challenging task categories. It offers full and mini distributions; this import contains the full distribution: 4,520 questions across all 23 task categories.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/google-deepmind/bbeh","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7aa7f29c-496c-53bb-87fe-68fdf1377cd5","task_key":"bbeh~5fcausal~5funderstanding--full--7aa7f29c-496c-53bb-87fe-68fdf1377cd5","task_revision_id":"1","upstream_id":"","short_description":"Question: Sam intends to add Reagent X to a mixture that would convert it to a…","config":"bbeh_causal_understanding","split":"full","body":"{\"input\":\"Question: Sam intends to add Reagent X to a mixture that would convert it to a crystal. Riya intends to prevent Sam from adding the reagent. Riya tries to stop Sam from adding the reagent, but Riya is stopped by Frank. Sam adds the reagent and the crystal is formed. Is Sam adding Reagent X to the mixture a sufficient cause for the crystal formation? Reply based on the answer a logician would give.\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/google-deepmind/bbeh","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}