{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"b1f50403-b8f5-56b2-9e9a-7303db565474","task_key":"web~5fof~5flies--main--b1f50403-b8f5-56b2-9e9a-7303db565474","task_revision_id":"1","upstream_id":"","short_description":"Question: Fletcher lies. Phoebe says Fletcher tells the truth. Vina says Phoebe…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Fletcher lies. Phoebe says Fletcher tells the truth. Vina says Phoebe tells the truth. Andree says Vina tells the truth. Willian says Andree tells the truth. Does Willian tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}