{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"arc-challenge","formal_name":"ARC-Challenge","introduction":"ARC-Challenge is the subset of grade-school science questions that both a retrieval baseline and a word co-occurrence baseline answered incorrectly. It tests reasoning with knowledge rather than recall alone.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/allenai/ai2_arc","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"44bce383-542d-5950-a9ae-8c2205146a66","task_key":"ARC~2dChallenge--test--Mercury~5f7213413","task_revision_id":"1","upstream_id":"Mercury_7213413","short_description":"A group of science students used water, filter paper, and a hot plate to…","config":"ARC-Challenge","split":"test","body":"{\"choices\":\"['shake the mixture until the sand settles on the bottom', 'pour the mixture through the filter paper', 'heat the mixture on the hot plate', 'dissolve the salt by adding water to the mixture']\",\"question\":\"A group of science students used water, filter paper, and a hot plate to separate a mixture of salt and sand. Which of the following did the students most likely do first?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/allenai/ai2_arc","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}