{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"9293b7de-399a-5cbf-bb90-1f749c19f21c","task_key":"default--test--9293b7de-399a-5cbf-bb90-1f749c19f21c","task_revision_id":"1","upstream_id":"","short_description":"Who did John Chapple, an employee of Buckingham Palace, need to notify of the…","config":"default","split":"test","body":"{\"problem\":\"Who did John Chapple, an employee of Buckingham Palace, need to notify of the Queen's passing when Queen Elizabeth II died on September 8, 2022, as part of his official duties?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}