{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"2a887392-f75b-5a0a-87b2-d42b170deb7c","task_key":"default--test--2292","task_revision_id":"1","upstream_id":"2292","short_description":"Which of the following reinforcement schedules results in maintenance of…","config":"default","split":"test","body":"{\"category\":\"psychology\",\"options\":\"['variable ratio', 'variable interval', 'fixed ratio with large ratio', 'continuous with long intervals', 'continuous', 'fixed ratio', 'fixed interval', 'variable ratio with large ratio', 'variable ratio with small ratio', 'fixed interval with long intervals']\",\"question\":\"Which of the following reinforcement schedules results in maintenance of behavior that is LEAST resistant to extinction?\",\"src\":\"ori_mmlu-high_school_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}