{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"mmlu-pro","formal_name":"MMLU-Pro","introduction":"MMLU-Pro rebuilds MMLU with ten answer options instead of four and removes items that no longer separate models. Questions span fourteen academic and professional subjects.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7b45aecc-0615-5940-993a-b6192befe04f","task_key":"default--test--5877","task_revision_id":"1","upstream_id":"5877","short_description":"Goldthorpe identified the 'service class' as:","config":"default","split":"test","body":"{\"category\":\"other\",\"options\":\"['the young men and women employed in domestic service in the nineteenth century', 'those in non-manual occupations, exercising authority on behalf of the state', 'those in manual labor jobs such as construction or manufacturing', 'individuals in skilled trades like plumbing, electricians, and carpentry', 'professionals in the arts, entertainment, and media industry', 'those who are self-employed or run their own businesses', 'people working in consultancy firms who were recruited by big businesses', 'those who had worked in the armed services']\",\"question\":\"Goldthorpe identified the 'service class' as:\",\"src\":\"ori_mmlu-sociology\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}