{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"simpleqa","formal_name":"SimpleQA","introduction":"SimpleQA collects short fact-seeking questions that have exactly one indisputable answer, measuring both how often a model is right and how often it declines. The format is deliberately narrow so that grading is not itself contestable.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://github.com/openai/simple-evals","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"11f7f63f-f00b-5c8d-b085-0a84442a0fa6","task_key":"default--test--11f7f63f-f00b-5c8d-b085-0a84442a0fa6","task_revision_id":"1","upstream_id":"","short_description":"According to the 14th report of the state entomologist on injurious and other…","config":"default","split":"test","body":"{\"problem\":\"According to the 14th report of the state entomologist on injurious and other insects of New York in 1898, Miss Ormerod, in her 15th report in 1893, recorded the serious and widespread injuries to raspberries from what insect in England? Use the scientific name.\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://github.com/openai/simple-evals","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}