{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a29f0ce0-2e5d-5554-b65b-083ee4d8eb6c","task_key":"default--test--a29f0ce0-2e5d-5554-b65b-083ee4d8eb6c","task_revision_id":"1","upstream_id":"","short_description":"What were the day, month, and year when Twitter overhauled its website once more…","config":"default","split":"test","body":"{\"problem\":\"What were the day, month, and year when Twitter overhauled its website once more to feature the \\\"Fly\\\" design, which the service says is easier for new users to follow and promotes advertising?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}