{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"humaneval","formal_name":"HumanEval","introduction":"HumanEval asks for a Python function body from a docstring and signature, checked by hidden unit tests. It is the original functional-correctness benchmark for code generation.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/openai/openai_humaneval","indexing_mode":"noindex","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[]}},"task_id":"1faa5b9f-31df-5408-acb2-131e5be25680","task_key":"openai~5fhumaneval--test--1faa5b9f-31df-5408-acb2-131e5be25680","task_revision_id":"1","upstream_id":"","short_description":"def solve(s):","config":"openai_humaneval","split":"test","body":"{\"entry_point\":\"solve\",\"prompt\":\"\\ndef solve(s):\\n    \\\"\\\"\\\"You are given a string s.\\n    if s[i] is a letter, reverse its case from lower to upper or vise versa, \\n    otherwise keep it as it is.\\n    If the string contains no letters, reverse the string.\\n    The function should return the resulted string.\\n    Examples\\n    solve(\\\"1234\\\") = \\\"4321\\\"\\n    solve(\\\"ab\\\") = \\\"AB\\\"\\n    solve(\\\"#a@C\\\") = \\\"#A@c\\\"\\n    \\\"\\\"\\\"\\n\"}","display_format":"text","language":"","answer_status":"unknown","assets":[],"source_url":"https://huggingface.co/datasets/openai/openai_humaneval","history":"initial import","indexing_mode":"noindex","subproblems":[],"grids":[]}