{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"bed3acea-b576-5cb3-93f6-d8831002b6ce","task_key":"web~5fof~5flies--main--bed3acea-b576-5cb3-93f6-d8831002b6ce","task_revision_id":"1","upstream_id":"","short_description":"Question: Lorine tells the truth. Michaela says Lorine lies. Millicent says…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Lorine tells the truth. Michaela says Lorine lies. Millicent says Michaela tells the truth. Fletcher says Millicent tells the truth. Shaunda says Fletcher tells the truth. Does Shaunda tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}