{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"613b63ea-5cb0-520a-9387-2c75b5df436e","task_key":"default--test--613b63ea-5cb0-520a-9387-2c75b5df436e","task_revision_id":"1","upstream_id":"","short_description":"What was the first line after the salutation in the letter sent to Sardar Patel…","config":"default","split":"test","body":"{\"problem\":\"What was the first line after the salutation in the letter sent to Sardar Patel by Abhay Charan De, also known as A. C. Bhaktivedanta Swami Prabhupada, on February 28, 1949?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}