diff --git a/tests/files/mock_responses/tasks/task_126033.xml b/tests/files/mock_responses/tasks/task_126033.xml new file mode 100644 index 000000000..2f07588ec --- /dev/null +++ b/tests/files/mock_responses/tasks/task_126033.xml @@ -0,0 +1,25 @@ + + 126033 + Task 126033: anneal (Clustering) + 5 + Clustering + + +2 + + + + +17 +testontrainingdata +https://openml.org/api_splits/get/126033/Task_126033_splits.arff +50 + + + + +ARFF + + + + diff --git a/tests/files/mock_responses/tasks/task_34536.xml b/tests/files/mock_responses/tasks/task_34536.xml new file mode 100644 index 000000000..48e8659c0 --- /dev/null +++ b/tests/files/mock_responses/tasks/task_34536.xml @@ -0,0 +1,39 @@ + + 34536 + Task 34536: Internet-Advertisements (Supervised Classification) + 1 + Supervised Classification + + +1176 +class + + + +1 +crossvalidation +https://openml.org/api_splits/get/34536/Task_34536_splits.arff +1 +10 + +true + + + + + + + + + +ARFF + + + + + + + OpenML100 + study_123 + study_14 + diff --git a/tests/files/mock_responses/tasks/task_5001.xml b/tests/files/mock_responses/tasks/task_5001.xml new file mode 100644 index 000000000..b5a5399cd --- /dev/null +++ b/tests/files/mock_responses/tasks/task_5001.xml @@ -0,0 +1,33 @@ + + 5001 + Task 5001: visualizing_galaxy (Supervised Regression) + 2 + Supervised Regression + + +690 +velocity + + + +7 +crossvalidation +https://openml.org//api_splits/get/5001/Task_5001_splits.arff +1 +10 + + + + +mean_absolute_error + + + +ARFF + + + + + + under1m + diff --git a/tests/files/mock_responses/tasks/task_64.xml b/tests/files/mock_responses/tasks/task_64.xml new file mode 100644 index 000000000..7b7b43a4d --- /dev/null +++ b/tests/files/mock_responses/tasks/task_64.xml @@ -0,0 +1,35 @@ + + 64 + Task 64: labor (Learning Curve) + 3 + Learning Curve + + +4 +class + + + +14 +crossvalidation +https://openml.org//api_splits/get/64/Task_64_splits.arff +10 +10 +1 + + + +predictive_accuracy + + + +ARFF + + + + + + + + under1m + diff --git a/tests/files/mock_responses/tasks/task_list_clustering.xml b/tests/files/mock_responses/tasks/task_list_clustering.xml new file mode 100644 index 000000000..d71291ad3 --- /dev/null +++ b/tests/files/mock_responses/tasks/task_list_clustering.xml @@ -0,0 +1,209 @@ + + + 126033 + 5 + Clustering + 2 + anneal + active + ARFF + 17 + 2 + 684.0 + 7.0 + 8.0 + 5.0 + 39.0 + 898.0 + 898.0 + 22175.0 + 6.0 + 33.0 + + + 126034 + 5 + Clustering + 3 + kr-vs-kp + active + ARFF + 17 + 3 + 1669.0 + 3.0 + 1527.0 + 2.0 + 37.0 + 3196.0 + 0.0 + 0.0 + 0.0 + 37.0 + + + 126035 + 5 + Clustering + 4 + labor + active + ARFF + 17 + 4 + 37.0 + 3.0 + 20.0 + 2.0 + 17.0 + 57.0 + 56.0 + 326.0 + 8.0 + 9.0 + + + 126036 + 5 + Clustering + 5 + arrhythmia + active + ARFF + 17 + 5 + 245.0 + 13.0 + 2.0 + 13.0 + 280.0 + 452.0 + 384.0 + 408.0 + 206.0 + 74.0 + + + 126037 + 5 + Clustering + 6 + letter + active + ARFF + 17 + 6 + 813.0 + 26.0 + 734.0 + 26.0 + 17.0 + 20000.0 + 0.0 + 0.0 + 16.0 + 1.0 + + + 126038 + 5 + Clustering + 7 + audiology + active + ARFF + 17 + 7 + 57.0 + 24.0 + 1.0 + 24.0 + 70.0 + 226.0 + 222.0 + 317.0 + 0.0 + 70.0 + + + 126039 + 5 + Clustering + 8 + liver-disorders + active + ARFF + 17 + 8 + 0.0 + 6.0 + 345.0 + 0.0 + 0.0 + 6.0 + 0.0 + + + 126040 + 5 + Clustering + 9 + autos + active + ARFF + 17 + 9 + 67.0 + 22.0 + 3.0 + 6.0 + 26.0 + 205.0 + 46.0 + 59.0 + 15.0 + 11.0 + + + 126041 + 5 + Clustering + 10 + lymph + active + ARFF + 17 + 10 + 81.0 + 8.0 + 2.0 + 4.0 + 19.0 + 148.0 + 0.0 + 0.0 + 3.0 + 16.0 + + + 126042 + 5 + Clustering + 11 + balance-scale + active + ARFF + 17 + 11 + 288.0 + 3.0 + 49.0 + 3.0 + 5.0 + 625.0 + 0.0 + 0.0 + 4.0 + 1.0 + + diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 80b0b4215..abcf94df0 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -140,10 +140,11 @@ def test_list_datasets_empty(self): datasets = openml.datasets.list_datasets(tag="NoOneWouldUseThisTagAnyway") assert datasets.empty - @pytest.mark.production_server() - def test_check_datasets_active(self): - # Have to test on live because there is no deactivated dataset on the test server. - self.use_production_server() + @mock.patch("openml.datasets.functions.list_datasets") + def test_check_datasets_active(self, mock_list_datasets): + mock_list_datasets.return_value = pd.DataFrame( + {"status": ["active", "deactivated"]}, index=[2, 17] + ) active = openml.datasets.check_datasets_active( [2, 17, 79], raise_error_if_not_exist=False, @@ -178,29 +179,37 @@ def test_illegal_length_tag(self): except openml.exceptions.OpenMLServerException as e: assert e.code == 477 - @pytest.mark.production_server() - def test__name_to_id_with_deactivated(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_deactivated(self, mock_list_datasets): """Check that an activated dataset is returned if an earlier deactivated one exists.""" - self.use_production_server() - # /d/1 was deactivated + # /d/1 was deactivated, /d/2 is active + mock_list_datasets.return_value = pd.DataFrame( + {"did": [2], "status": ["active"]}, index=[2] + ) assert openml.datasets.functions._name_to_id("anneal") == 2 - @pytest.mark.production_server() - def test__name_to_id_with_multiple_active(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_multiple_active(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [61, 969], "status": ["active", "active"]}, index=[61, 969] + ) assert openml.datasets.functions._name_to_id("iris") == 61 - @pytest.mark.production_server() - def test__name_to_id_with_version(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_version(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [969], "status": ["active"]}, index=[969] + ) assert openml.datasets.functions._name_to_id("iris", version=3) == 969 - @pytest.mark.production_server() - def test__name_to_id_with_multiple_active_error(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_multiple_active_error(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [61, 969], "status": ["active", "active"]}, index=[61, 969] + ) self.assertRaisesRegex( ValueError, "Multiple active datasets exist with name 'iris'.", @@ -280,16 +289,16 @@ def test_get_dataset_uint8_dtype(self): df, _, _, _ = dataset.get_data() assert df["carbon"].dtype == "uint8" - @pytest.mark.production_server() - def test_get_dataset_cannot_access_private_data(self): + @mock.patch("openml._api_calls._perform_api_call") + def test_get_dataset_cannot_access_private_data(self, mock_api): # Issue324 Properly handle private datasets when trying to access them - self.use_production_server() + mock_api.side_effect = OpenMLPrivateDatasetError("No access granted") self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, 45) - @pytest.mark.skip("Need to find dataset name of private dataset") - def test_dataset_by_name_cannot_access_private_data(self): - self.use_production_server() - self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE") + @mock.patch("openml._api_calls._perform_api_call") + def test_dataset_by_name_cannot_access_private_data(self, mock_api): + mock_api.side_effect = OpenMLPrivateDatasetError("No access granted") + self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "private_dataset") @pytest.mark.test_server() def test_get_dataset_lazy_all_functions(self): @@ -1541,15 +1550,15 @@ def test_data_fork(self): ) - @pytest.mark.production_server() - def test_list_datasets_with_high_size_parameter(self): - # Testing on prod since concurrent deletion of uploded datasets make the test fail - self.use_production_server() + @mock.patch("openml.datasets.functions._list_datasets") + def test_list_datasets_with_high_size_parameter(self, mock_inner_list): + # Testing that size=np.inf produces the same call as default (no size limit) + fake_df = pd.DataFrame({"did": [1, 2, 3], "status": ["active"] * 3}) + mock_inner_list.return_value = fake_df datasets_a = openml.datasets.list_datasets() datasets_b = openml.datasets.list_datasets(size=np.inf) - # Reverting to test server assert len(datasets_a) == len(datasets_b) diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index bf2fcfeae..b729a49ed 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -2,6 +2,7 @@ from __future__ import annotations import os +import pathlib import unittest from typing import cast from unittest import mock @@ -56,11 +57,17 @@ def test__get_estimation_procedure_list(self): assert isinstance(estimation_procedures[0], dict) assert estimation_procedures[0]["task_type_id"] == TaskType.SUPERVISED_CLASSIFICATION - @pytest.mark.production_server() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) - def test_list_clustering_task(self): - self.use_production_server() + @mock.patch("requests.Session.request") + def test_list_clustering_task(self, mock_request): # as shown by #383, clustering tasks can give list/dict casting problems + mock_request.return_value = create_request_response( + status_code=200, + content_filepath=pathlib.Path(__file__).parent.parent + / "files" + / "mock_responses" + / "tasks" + / "task_list_clustering.xml", + ) openml.tasks.list_tasks(task_type=TaskType.CLUSTERING, size=10) # the expected outcome is that it doesn't crash. No assertions. @@ -143,15 +150,19 @@ def test__get_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.tasks.get_task(1882) - @unittest.skip( - "Please await outcome of discussion: https://github.com/openml/OpenML/issues/776", - ) - @pytest.mark.production_server() - def test__get_task_live(self): - self.use_production_server() + @mock.patch("openml._api_calls._perform_api_call") + def test__get_task_live(self, mock_api): # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 - openml.tasks.get_task(34536) + task_xml = ( + pathlib.Path(__file__).parent.parent + / "files" + / "mock_responses" + / "tasks" + / "task_34536.xml" + ).read_text() + mock_api.return_value = task_xml + openml.tasks.functions._get_task_description(34536) @pytest.mark.test_server() def test_get_task(self): @@ -214,15 +225,29 @@ def test_get_task_with_cache(self): task = openml.tasks.get_task(1) assert isinstance(task, OpenMLTask) - @pytest.mark.production_server() - def test_get_task_different_types(self): - self.use_production_server() + @mock.patch("openml._api_calls._perform_api_call") + def test_get_task_different_types(self, mock_api): + mock_responses_dir = ( + pathlib.Path(__file__).parent.parent + / "files" + / "mock_responses" + / "tasks" + ) + + def side_effect(call, method, **kwargs): + # Map API call to the corresponding mock XML file + task_id = call.split("/")[-1] + xml_path = mock_responses_dir / f"task_{task_id}.xml" + return xml_path.read_text() + + mock_api.side_effect = side_effect + # Regression task - openml.tasks.functions.get_task(5001) + openml.tasks.functions._get_task_description(5001) # Learning curve - openml.tasks.functions.get_task(64) + openml.tasks.functions._get_task_description(64) # Issue 538, get_task failing with clustering task. - openml.tasks.functions.get_task(126033) + openml.tasks.functions._get_task_description(126033) @pytest.mark.test_server() def test_download_split(self):