{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"1e7ae4d5-4b60-5ac2-81ec-baaecac43509","task_key":"default--test--1e7ae4d5-4b60-5ac2-81ec-baaecac43509","task_revision_id":"1","upstream_id":"","short_description":"When diver Vicki Manalo joined the swimming program at the Crystal Plunge in…","config":"default","split":"test","body":"{\"problem\":\"When diver Vicki Manalo joined the swimming program at the Crystal Plunge in North Beach, San Francisco, CA, what were the first and last names of the man assigned as her coach?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}