{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1e96095f-2260-57fb-acc4-c20ac5c501b2","task_key":"web~5fof~5flies--main--1e96095f-2260-57fb-acc4-c20ac5c501b2","task_revision_id":"1","upstream_id":"","short_description":"Question: Millicent tells the truth. Millie says Millicent tells the truth.…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Millicent tells the truth. Millie says Millicent tells the truth. Michael says Millie tells the truth. Antwan says Michael lies. Alexis says Antwan tells the truth. Does Alexis tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}