{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2e12c970-c3b1-544b-8e5e-2c06add66ba4","task_key":"default--test--2e12c970-c3b1-544b-8e5e-2c06add66ba4","task_revision_id":"1","upstream_id":"","short_description":"What time (hours, minutes, and seconds) did Lawrence Whittaker finish in when he…","config":"default","split":"test","body":"{\"problem\":\"What time (hours, minutes, and seconds) did Lawrence Whittaker finish in when he won the first Cape Town Cycle Tour, formerly known as the Cape Argus Cycle Tour, in September 1978?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}