{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"86a77cc8-3868-5b0b-8f42-c29e328772cd","task_key":"default--test--7887","task_revision_id":"1","upstream_id":"7887","short_description":"An urn contains four colored balls: two orange and two blue. Two balls are…","config":"default","split":"test","body":"{\"category\":\"math\",\"options\":\"['$\\\\\\\\frac{3}{4}$', '$\\\\\\\\frac{3}{5}$', '$\\\\\\\\frac{1}{4}$', '$\\\\\\\\frac{1}{2}$', '$\\\\\\\\frac{1}{6}$', '$\\\\\\\\frac{1}{5}$', '$\\\\\\\\frac{2}{5}$', '$\\\\\\\\frac{1}{3}$', '$\\\\\\\\frac{2}{3}$', '$\\\\\\\\frac{5}{6}$']\",\"question\":\"An urn contains four colored balls: two orange and two blue. Two balls are selected at random without replacement, and you are told that at least one of them is orange. What is the probability that the other ball is also orange?\",\"src\":\"scibench-stat\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}