{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"f0cf7f7a-e52a-50b6-9fa0-2f496139fba4","task_key":"default--test--f0cf7f7a-e52a-50b6-9fa0-2f496139fba4","task_revision_id":"1","upstream_id":"","short_description":"When the electric street lamp contract in Paris, Ontario, with Orlande H.…","config":"default","split":"test","body":"{\"problem\":\"When the electric street lamp contract in Paris, Ontario, with Orlande H. Duncombe and Alonzo N. Parney expired in 1887, what company agreed to light 25 lamps until 12 p.m. for 26 cents a lamp per night?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}