{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b018c49e-ff7f-5d49-bb7c-c9ee4550232f","task_key":"web~5fof~5flies--main--b018c49e-ff7f-5d49-bb7c-c9ee4550232f","task_revision_id":"1","upstream_id":"","short_description":"Question: Osvaldo tells the truth. Andree says Osvaldo tells the truth. Willian…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Osvaldo tells the truth. Andree says Osvaldo tells the truth. Willian says Andree lies. Vina says Willian tells the truth. Jaymie says Vina tells the truth. Does Jaymie tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}