{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ebe89a5f-af28-5e56-b581-f727ca72322b","task_key":"web~5fof~5flies--main--ebe89a5f-af28-5e56-b581-f727ca72322b","task_revision_id":"1","upstream_id":"","short_description":"Question: Tamika tells the truth. Ka says Tamika tells the truth. Fidel says Ka…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Tamika tells the truth. Ka says Tamika tells the truth. Fidel says Ka tells the truth. Elanor says Fidel tells the truth. Amberly says Elanor tells the truth. Does Amberly tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}