{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"77dbb9e2-0d66-5435-90e3-a79c327a96e7","task_key":"default--test--5342","task_revision_id":"1","upstream_id":"5342","short_description":"A practitioner is engaged to express an opinion on management’s assertion that…","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['Statements on Standards for Personal Financial Planning Services', 'Statements on Standards for Accounting and Review Services', 'Statements on Standards for Financial Advisory Services', 'Statements on Standards for Forensic Accounting Services', 'Statements on Standards for Information Technology Services', 'Statements on Standards for Tax Services', 'Statements on Standards for Consulting Services', 'Statements on Auditing Standards', 'Statements on Standards for Business Valuation Services', 'Statements on Standards for Attestation Engagements']\",\"question\":\"A practitioner is engaged to express an opinion on management’s assertion that the square footage of a warehouse offered for sale is 150,000 square feet. The practitioner should refer to which of the following sources for professional guidance?\",\"src\":\"ori_mmlu-professional_accounting\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}