From 2314cbca4656976222e7cebcab6db7114a33cd62 Mon Sep 17 00:00:00 2001 From: Sumit Saraswat Date: Sat, 3 Oct 2026 18:58:39 +0530 Subject: [PATCH 1/3] test: mock live server call in test_list_clustering_task Replaces the @pytest.mark.production_server() call in test_list_clustering_task with a mocked requests.Session.request using a saved XML response. Partial fix for #1649. --- tests/mock_data/clustering_tasks.xml | 209 ++++++++++++++++++++++++ tests/test_tasks/test_task_functions.py | 13 +- 2 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 tests/mock_data/clustering_tasks.xml diff --git a/tests/mock_data/clustering_tasks.xml b/tests/mock_data/clustering_tasks.xml new file mode 100644 index 000000000..d71291ad3 --- /dev/null +++ b/tests/mock_data/clustering_tasks.xml @@ -0,0 +1,209 @@ + + + 126033 + 5 + Clustering + 2 + anneal + active + ARFF + 17 + 2 + 684.0 + 7.0 + 8.0 + 5.0 + 39.0 + 898.0 + 898.0 + 22175.0 + 6.0 + 33.0 + + + 126034 + 5 + Clustering + 3 + kr-vs-kp + active + ARFF + 17 + 3 + 1669.0 + 3.0 + 1527.0 + 2.0 + 37.0 + 3196.0 + 0.0 + 0.0 + 0.0 + 37.0 + + + 126035 + 5 + Clustering + 4 + labor + active + ARFF + 17 + 4 + 37.0 + 3.0 + 20.0 + 2.0 + 17.0 + 57.0 + 56.0 + 326.0 + 8.0 + 9.0 + + + 126036 + 5 + Clustering + 5 + arrhythmia + active + ARFF + 17 + 5 + 245.0 + 13.0 + 2.0 + 13.0 + 280.0 + 452.0 + 384.0 + 408.0 + 206.0 + 74.0 + + + 126037 + 5 + Clustering + 6 + letter + active + ARFF + 17 + 6 + 813.0 + 26.0 + 734.0 + 26.0 + 17.0 + 20000.0 + 0.0 + 0.0 + 16.0 + 1.0 + + + 126038 + 5 + Clustering + 7 + audiology + active + ARFF + 17 + 7 + 57.0 + 24.0 + 1.0 + 24.0 + 70.0 + 226.0 + 222.0 + 317.0 + 0.0 + 70.0 + + + 126039 + 5 + Clustering + 8 + liver-disorders + active + ARFF + 17 + 8 + 0.0 + 6.0 + 345.0 + 0.0 + 0.0 + 6.0 + 0.0 + + + 126040 + 5 + Clustering + 9 + autos + active + ARFF + 17 + 9 + 67.0 + 22.0 + 3.0 + 6.0 + 26.0 + 205.0 + 46.0 + 59.0 + 15.0 + 11.0 + + + 126041 + 5 + Clustering + 10 + lymph + active + ARFF + 17 + 10 + 81.0 + 8.0 + 2.0 + 4.0 + 19.0 + 148.0 + 0.0 + 0.0 + 3.0 + 16.0 + + + 126042 + 5 + Clustering + 11 + balance-scale + active + ARFF + 17 + 11 + 288.0 + 3.0 + 49.0 + 3.0 + 5.0 + 625.0 + 0.0 + 0.0 + 4.0 + 1.0 + + diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index bf2fcfeae..9b11513d7 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -56,10 +56,15 @@ def test__get_estimation_procedure_list(self): assert isinstance(estimation_procedures[0], dict) assert estimation_procedures[0]["task_type_id"] == TaskType.SUPERVISED_CLASSIFICATION - @pytest.mark.production_server() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) - def test_list_clustering_task(self): - self.use_production_server() + @mock.patch("requests.Session.request") + def test_list_clustering_task(self, mock_request): + import pathlib + mock_request.return_value = create_request_response( + status_code=200, + content_filepath=pathlib.Path(__file__).parent.parent + / "mock_data" + / "clustering_tasks.xml", + ) # as shown by #383, clustering tasks can give list/dict casting problems openml.tasks.list_tasks(task_type=TaskType.CLUSTERING, size=10) # the expected outcome is that it doesn't crash. No assertions. From 3a405481116a077a2015100b9b38733b87523708 Mon Sep 17 00:00:00 2001 From: Sumit Saraswat Date: Sat, 3 Oct 2026 20:24:00 +0530 Subject: [PATCH 2/3] test: mock all production server calls in test_task_functions.py Replace all three @pytest.mark.production_server() tests with mocked versions that use saved XML responses: - test_list_clustering_task: mock requests.Session.request with saved clustering task list XML - test__get_task_live: mock _perform_api_call with saved task 34536 XML, also remove @unittest.skip since the mock makes it runnable - test_get_task_different_types: mock _perform_api_call with saved XMLs for regression (5001), learning curve (64), and clustering (126033) Mock XML files are placed in tests/files/mock_responses/tasks/ following the existing project convention. Partial fix for #1649. --- .../mock_responses/tasks/task_126033.xml | 25 +++++++++ .../files/mock_responses/tasks/task_34536.xml | 39 ++++++++++++++ .../files/mock_responses/tasks/task_5001.xml | 33 ++++++++++++ tests/files/mock_responses/tasks/task_64.xml | 35 ++++++++++++ .../tasks/task_list_clustering.xml} | 0 tests/test_tasks/test_task_functions.py | 54 +++++++++++++------ 6 files changed, 169 insertions(+), 17 deletions(-) create mode 100644 tests/files/mock_responses/tasks/task_126033.xml create mode 100644 tests/files/mock_responses/tasks/task_34536.xml create mode 100644 tests/files/mock_responses/tasks/task_5001.xml create mode 100644 tests/files/mock_responses/tasks/task_64.xml rename tests/{mock_data/clustering_tasks.xml => files/mock_responses/tasks/task_list_clustering.xml} (100%) diff --git a/tests/files/mock_responses/tasks/task_126033.xml b/tests/files/mock_responses/tasks/task_126033.xml new file mode 100644 index 000000000..2f07588ec --- /dev/null +++ b/tests/files/mock_responses/tasks/task_126033.xml @@ -0,0 +1,25 @@ + + 126033 + Task 126033: anneal (Clustering) + 5 + Clustering + + +2 + + + + +17 +testontrainingdata +https://openml.org/api_splits/get/126033/Task_126033_splits.arff +50 + + + + +ARFF + + + + diff --git a/tests/files/mock_responses/tasks/task_34536.xml b/tests/files/mock_responses/tasks/task_34536.xml new file mode 100644 index 000000000..48e8659c0 --- /dev/null +++ b/tests/files/mock_responses/tasks/task_34536.xml @@ -0,0 +1,39 @@ + + 34536 + Task 34536: Internet-Advertisements (Supervised Classification) + 1 + Supervised Classification + + +1176 +class + + + +1 +crossvalidation +https://openml.org/api_splits/get/34536/Task_34536_splits.arff +1 +10 + +true + + + + + + + + + +ARFF + + + + + + + OpenML100 + study_123 + study_14 + diff --git a/tests/files/mock_responses/tasks/task_5001.xml b/tests/files/mock_responses/tasks/task_5001.xml new file mode 100644 index 000000000..b5a5399cd --- /dev/null +++ b/tests/files/mock_responses/tasks/task_5001.xml @@ -0,0 +1,33 @@ + + 5001 + Task 5001: visualizing_galaxy (Supervised Regression) + 2 + Supervised Regression + + +690 +velocity + + + +7 +crossvalidation +https://openml.org//api_splits/get/5001/Task_5001_splits.arff +1 +10 + + + + +mean_absolute_error + + + +ARFF + + + + + + under1m + diff --git a/tests/files/mock_responses/tasks/task_64.xml b/tests/files/mock_responses/tasks/task_64.xml new file mode 100644 index 000000000..7b7b43a4d --- /dev/null +++ b/tests/files/mock_responses/tasks/task_64.xml @@ -0,0 +1,35 @@ + + 64 + Task 64: labor (Learning Curve) + 3 + Learning Curve + + +4 +class + + + +14 +crossvalidation +https://openml.org//api_splits/get/64/Task_64_splits.arff +10 +10 +1 + + + +predictive_accuracy + + + +ARFF + + + + + + + + under1m + diff --git a/tests/mock_data/clustering_tasks.xml b/tests/files/mock_responses/tasks/task_list_clustering.xml similarity index 100% rename from tests/mock_data/clustering_tasks.xml rename to tests/files/mock_responses/tasks/task_list_clustering.xml diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 9b11513d7..b729a49ed 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -2,6 +2,7 @@ from __future__ import annotations import os +import pathlib import unittest from typing import cast from unittest import mock @@ -58,14 +59,15 @@ def test__get_estimation_procedure_list(self): @mock.patch("requests.Session.request") def test_list_clustering_task(self, mock_request): - import pathlib + # as shown by #383, clustering tasks can give list/dict casting problems mock_request.return_value = create_request_response( status_code=200, content_filepath=pathlib.Path(__file__).parent.parent - / "mock_data" - / "clustering_tasks.xml", + / "files" + / "mock_responses" + / "tasks" + / "task_list_clustering.xml", ) - # as shown by #383, clustering tasks can give list/dict casting problems openml.tasks.list_tasks(task_type=TaskType.CLUSTERING, size=10) # the expected outcome is that it doesn't crash. No assertions. @@ -148,15 +150,19 @@ def test__get_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.tasks.get_task(1882) - @unittest.skip( - "Please await outcome of discussion: https://github.com/openml/OpenML/issues/776", - ) - @pytest.mark.production_server() - def test__get_task_live(self): - self.use_production_server() + @mock.patch("openml._api_calls._perform_api_call") + def test__get_task_live(self, mock_api): # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 - openml.tasks.get_task(34536) + task_xml = ( + pathlib.Path(__file__).parent.parent + / "files" + / "mock_responses" + / "tasks" + / "task_34536.xml" + ).read_text() + mock_api.return_value = task_xml + openml.tasks.functions._get_task_description(34536) @pytest.mark.test_server() def test_get_task(self): @@ -219,15 +225,29 @@ def test_get_task_with_cache(self): task = openml.tasks.get_task(1) assert isinstance(task, OpenMLTask) - @pytest.mark.production_server() - def test_get_task_different_types(self): - self.use_production_server() + @mock.patch("openml._api_calls._perform_api_call") + def test_get_task_different_types(self, mock_api): + mock_responses_dir = ( + pathlib.Path(__file__).parent.parent + / "files" + / "mock_responses" + / "tasks" + ) + + def side_effect(call, method, **kwargs): + # Map API call to the corresponding mock XML file + task_id = call.split("/")[-1] + xml_path = mock_responses_dir / f"task_{task_id}.xml" + return xml_path.read_text() + + mock_api.side_effect = side_effect + # Regression task - openml.tasks.functions.get_task(5001) + openml.tasks.functions._get_task_description(5001) # Learning curve - openml.tasks.functions.get_task(64) + openml.tasks.functions._get_task_description(64) # Issue 538, get_task failing with clustering task. - openml.tasks.functions.get_task(126033) + openml.tasks.functions._get_task_description(126033) @pytest.mark.test_server() def test_download_split(self): From b38f7c3c02aa37ba8d6a51a795223f05f36fd722 Mon Sep 17 00:00:00 2001 From: Sumit Saraswat Date: Sat, 3 Oct 2026 20:40:09 +0530 Subject: [PATCH 3/3] test: mock all production server calls in test_dataset_functions.py Replace all seven @pytest.mark.production_server() tests with mocked versions using unittest.mock.patch: - test_check_datasets_active: mock list_datasets with fake DataFrame - test__name_to_id_with_deactivated: mock list_datasets - test__name_to_id_with_multiple_active: mock list_datasets - test__name_to_id_with_version: mock list_datasets - test__name_to_id_with_multiple_active_error: mock list_datasets - test_get_dataset_cannot_access_private_data: mock _perform_api_call to raise OpenMLPrivateDatasetError - test_dataset_by_name_cannot_access_private_data: same mock, also removed @pytest.mark.skip since the mock makes it runnable - test_list_datasets_with_high_size_parameter: mock _list_datasets After this PR, test_dataset_functions.py has zero production_server markers remaining. Partial fix for #1649. --- tests/test_datasets/test_dataset_functions.py | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 80b0b4215..abcf94df0 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -140,10 +140,11 @@ def test_list_datasets_empty(self): datasets = openml.datasets.list_datasets(tag="NoOneWouldUseThisTagAnyway") assert datasets.empty - @pytest.mark.production_server() - def test_check_datasets_active(self): - # Have to test on live because there is no deactivated dataset on the test server. - self.use_production_server() + @mock.patch("openml.datasets.functions.list_datasets") + def test_check_datasets_active(self, mock_list_datasets): + mock_list_datasets.return_value = pd.DataFrame( + {"status": ["active", "deactivated"]}, index=[2, 17] + ) active = openml.datasets.check_datasets_active( [2, 17, 79], raise_error_if_not_exist=False, @@ -178,29 +179,37 @@ def test_illegal_length_tag(self): except openml.exceptions.OpenMLServerException as e: assert e.code == 477 - @pytest.mark.production_server() - def test__name_to_id_with_deactivated(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_deactivated(self, mock_list_datasets): """Check that an activated dataset is returned if an earlier deactivated one exists.""" - self.use_production_server() - # /d/1 was deactivated + # /d/1 was deactivated, /d/2 is active + mock_list_datasets.return_value = pd.DataFrame( + {"did": [2], "status": ["active"]}, index=[2] + ) assert openml.datasets.functions._name_to_id("anneal") == 2 - @pytest.mark.production_server() - def test__name_to_id_with_multiple_active(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_multiple_active(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [61, 969], "status": ["active", "active"]}, index=[61, 969] + ) assert openml.datasets.functions._name_to_id("iris") == 61 - @pytest.mark.production_server() - def test__name_to_id_with_version(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_version(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [969], "status": ["active"]}, index=[969] + ) assert openml.datasets.functions._name_to_id("iris", version=3) == 969 - @pytest.mark.production_server() - def test__name_to_id_with_multiple_active_error(self): + @mock.patch("openml.datasets.functions.list_datasets") + def test__name_to_id_with_multiple_active_error(self, mock_list_datasets): """With multiple active datasets, retrieve the least recent active.""" - self.use_production_server() + mock_list_datasets.return_value = pd.DataFrame( + {"did": [61, 969], "status": ["active", "active"]}, index=[61, 969] + ) self.assertRaisesRegex( ValueError, "Multiple active datasets exist with name 'iris'.", @@ -280,16 +289,16 @@ def test_get_dataset_uint8_dtype(self): df, _, _, _ = dataset.get_data() assert df["carbon"].dtype == "uint8" - @pytest.mark.production_server() - def test_get_dataset_cannot_access_private_data(self): + @mock.patch("openml._api_calls._perform_api_call") + def test_get_dataset_cannot_access_private_data(self, mock_api): # Issue324 Properly handle private datasets when trying to access them - self.use_production_server() + mock_api.side_effect = OpenMLPrivateDatasetError("No access granted") self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, 45) - @pytest.mark.skip("Need to find dataset name of private dataset") - def test_dataset_by_name_cannot_access_private_data(self): - self.use_production_server() - self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE") + @mock.patch("openml._api_calls._perform_api_call") + def test_dataset_by_name_cannot_access_private_data(self, mock_api): + mock_api.side_effect = OpenMLPrivateDatasetError("No access granted") + self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "private_dataset") @pytest.mark.test_server() def test_get_dataset_lazy_all_functions(self): @@ -1541,15 +1550,15 @@ def test_data_fork(self): ) - @pytest.mark.production_server() - def test_list_datasets_with_high_size_parameter(self): - # Testing on prod since concurrent deletion of uploded datasets make the test fail - self.use_production_server() + @mock.patch("openml.datasets.functions._list_datasets") + def test_list_datasets_with_high_size_parameter(self, mock_inner_list): + # Testing that size=np.inf produces the same call as default (no size limit) + fake_df = pd.DataFrame({"did": [1, 2, 3], "status": ["active"] * 3}) + mock_inner_list.return_value = fake_df datasets_a = openml.datasets.list_datasets() datasets_b = openml.datasets.list_datasets(size=np.inf) - # Reverting to test server assert len(datasets_a) == len(datasets_b)