{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"arc-challenge","formal_name":"ARC-Challenge","introduction":"ARC-Challenge is the subset of grade-school science questions that both a retrieval baseline and a word co-occurrence baseline answered incorrectly. It tests reasoning with knowledge rather than recall alone.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/allenai/ai2_arc","indexing_mode":"index","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[],"history":"","construction":"","limitations":"","saturation":"","scores":[]}},"task_id":"98c71ea2-9a79-5d43-bb70-6f282e536f22","task_key":"ARC~2dChallenge--test--Mercury~5f7064750","task_revision_id":"1","upstream_id":"Mercury_7064750","short_description":"A student pours salt water into a beaker and then pours some acid into another…","config":"ARC-Challenge","split":"test","body":"{\"choices\":\"['smell each sample in order to identify it', 'guess which beaker contains each substance', 'mix the substances together to determine what they are', 'safely discard the samples and get new ones']\",\"question\":\"A student pours salt water into a beaker and then pours some acid into another beaker. The student returns to the lab table and realizes that the samples are not marked. What should the student do?\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/allenai/ai2_arc","history":"initial import","indexing_mode":"index","subproblems":[],"grids":[]}