{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"bbh","formal_name":"BIG-Bench Hard","introduction":"BIG-Bench Hard is the set of 23 BIG-Bench tasks on which models of the time scored below the average human rater. It is best known for showing how much chain-of-thought prompting changes the result.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"6c986b01-8788-552b-9612-468b46a16def","task_key":"web~5fof~5flies--main--6c986b01-8788-552b-9612-468b46a16def","task_revision_id":"1","upstream_id":"","short_description":"Question: Jamey lies. Gwenn says Jamey tells the truth. Yoland says Gwenn tells…","config":"web_of_lies","split":"main","body":"{\"input\":\"Question: Jamey lies. Gwenn says Jamey tells the truth. Yoland says Gwenn tells the truth. Kandi says Yoland tells the truth. Osvaldo says Kandi tells the truth. Does Osvaldo tell the truth?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/suzgunmirac/BIG-Bench-Hard","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}