{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"ff410ac0-d2cc-5904-ad01-5a6b512d09f8","task_key":"default--test--10522","task_revision_id":"1","upstream_id":"10522","short_description":"A particular parallel program computation requires 100 seconds when executed on…","config":"default","split":"test","body":"{\"category\":\"computer science\",\"options\":\"['100 and 50 seconds', '40 and 20 seconds', '60 and 30 seconds', '20 and 10 seconds', '80 and 40 seconds', '70 and 55 seconds', '30 and 15 seconds', '50 and 25 seconds', '110 and 55 seconds', '90 and 45 seconds']\",\"question\":\"A particular parallel program computation requires 100 seconds when executed on a single processor. If 40 percent of this computation is \\\"inherently sequential\\\" (i.e., will not benefit from additional processors), then the theoretically best possible elapsed times for this program running with 2 and 4 processors, respectively, are\",\"src\":\"ori_mmlu-college_computer_science\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}