{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"a0975cc9-1c30-560d-9d15-2b449b7beb18","task_key":"default--test--2387","task_revision_id":"1","upstream_id":"2387","short_description":"A student has recently completed a dissertation and is submitting a draft for…","config":"default","split":"test","body":"{\"category\":\"psychology\",\"options\":\"['first authorship with the advisor as second author', 'co-first authorship with the advisor', 'first authorship with the advisor receiving acknowledgment in a footnote', 'firs authorship with the advisor receiving no acknowledgment', 'second authorship with the advisor as co-first author', 'second authorship with the advisor as first author', 'first authorship with the advisor as last author', 'first authorship with the advisor acknowledged in the acknowledgments section', 'second authorship with the advisor receiving acknowledgment in a footnote', 'sole authorship with no acknowledgment of the advisor']\",\"question\":\"A student has recently completed a dissertation and is submitting a draft for publication. The student’s advisor contributed by assisting with data analysis and writing a major part of the first publication draft. If the manuscript is published, the student should receive\",\"src\":\"ori_mmlu-professional_psychology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}