{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"arc-challenge","formal_name":"ARC-Challenge","introduction":"ARC-Challenge is the subset of grade-school science questions that both a retrieval baseline and a word co-occurrence baseline answered incorrectly. It tests reasoning with knowledge rather than recall alone.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/allenai/ai2_arc","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"7e814cda-7c71-591a-b7d5-308063662ba3","task_key":"ARC~2dChallenge--test--MEA~5f2016~5f8~5f15","task_revision_id":"1","upstream_id":"MEA_2016_8_15","short_description":"Which list has objects in order from the smallest to the largest?","config":"ARC-Challenge","split":"test","body":"{\"choices\":\"['Earth, the Sun, solar system, the Milky Way galaxy', 'the Moon, the Sun, Earth, the Milky Way galaxy', 'the Sun, the Moon, Earth, the Milky Way galaxy', 'solar system, the Milky Way galaxy, Earth, the Moon']\",\"question\":\"Which list has objects in order from the smallest to the largest?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/allenai/ai2_arc","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}