diff --git a/coriolis/constants.py b/coriolis/constants.py index 5624f130..56d7e6ed 100644 --- a/coriolis/constants.py +++ b/coriolis/constants.py @@ -274,6 +274,10 @@ VALID_OS_TYPES = [OS_TYPE_BSD, OS_TYPE_LINUX, OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS] +PROTOCOL_SSH = "ssh" +# WinRM is the Microsoft implementation of WSMAN. +PROTOCOL_WINRM = "winrm" + TMP_DIRS_KEY = "__tmp_dirs" COMPRESSION_FORMAT_GZIP = "gzip" diff --git a/coriolis/providers/backup_writers.py b/coriolis/providers/backup_writers.py index 357a2f9d..4cc9a41a 100644 --- a/coriolis/providers/backup_writers.py +++ b/coriolis/providers/backup_writers.py @@ -22,6 +22,7 @@ from coriolis import constants from coriolis import data_transfer +from coriolis import events from coriolis import exception from coriolis.providers import provider_utils from coriolis import utils @@ -51,6 +52,18 @@ BACKUP_WRITER_FILE ] +# Common data transfer mechanisms exposed to Coriolis users. +DATA_TRANSFER_MECHANISM_SSH = "SSH" +DATA_TRANSFER_MECHANISM_HTTPS = "HTTPS" +# Can't be the same as replicator port. +DATA_TRANSFER_MECHANISM_HTTPS_PORT = 5566 + +# The file writer is meant for testing purposes and will not be exposed here. +DATA_TRANSFER_MECHANISM_MAP = { + DATA_TRANSFER_MECHANISM_SSH: BACKUP_WRITER_SSH, + DATA_TRANSFER_MECHANISM_HTTPS: BACKUP_WRITER_HTTP, +} + _WRITER_ERR_MAP = { -1: "ERR_MORE_MSG", 0: "ERR_DONE", @@ -176,6 +189,54 @@ def _validate_info(self, info): raise exception.CoriolisException( "Missing credentials in connection info") + @classmethod + def get_backup_writer_connection_info( + cls, + event_manager: events.EventManager, + ssh_connection_info: dict, + data_transfer_mechanism: str, + ) -> dict: + """Initialize the backup writer and obtain connection info. + + :param ssh_connection_info: a dict containing the following keys: + * ip + * port - usually SSH port (22) + * username + * password + * pkey - Paramiko keypair + :param data_transfer_mechanism: SSH or HTTPS + :returns: a dict describing the backend type and backup writer + connection details, used to subsequently retrieve the + backup writer. + """ + if data_transfer_mechanism == DATA_TRANSFER_MECHANISM_HTTPS: + event_manager.progress_update( + "Setting up HTTPS backup writer service on disk copy worker VM" + ) + writer_bootstrapper = HTTPBackupWriterBootstrapper( + ssh_connection_info, + DATA_TRANSFER_MECHANISM_HTTPS_PORT, + ) + https_conn_info = writer_bootstrapper.setup_writer() + writer_conn_info = { + "backend": DATA_TRANSFER_MECHANISM_MAP[ + DATA_TRANSFER_MECHANISM_HTTPS], + "connection_details": https_conn_info, + } + elif data_transfer_mechanism == DATA_TRANSFER_MECHANISM_SSH: + writer_conn_info = { + "backend": DATA_TRANSFER_MECHANISM_MAP[ + DATA_TRANSFER_MECHANISM_SSH], + "connection_details": ssh_connection_info, + } + else: + raise ValueError( + "Unhandleable data transfer mechanism '%s'" % ( + data_transfer_mechanism) + ) + + return writer_conn_info + class BaseBackupWriterImpl(with_metaclass(abc.ABCMeta)): def __init__(self, path, disk_id): diff --git a/coriolis/providers/provider_utils.py b/coriolis/providers/provider_utils.py index e384683b..0d91187b 100644 --- a/coriolis/providers/provider_utils.py +++ b/coriolis/providers/provider_utils.py @@ -1,9 +1,15 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. -from oslo_log import log as logging import requests +import time + +from oslo_log import log as logging +import paramiko +from coriolis import constants from coriolis import exception +from coriolis import utils +from coriolis import wsman LOG = logging.getLogger(__name__) @@ -140,3 +146,98 @@ def merge_environment_settings( verify = self.verify return super(ProviderSession, self).merge_environment_settings( url, proxies, stream, verify, *args, **kwargs) + + +def _poll_instance_until_reachable_ssh( + connection_info: dict, + timeout: int = 600, + poll_interval: int = 10, +): + start = time.time() + while (time.time() - start) < timeout: + try: + ssh = utils.connect_ssh( + hostname=connection_info["ip"], + port=connection_info["port"], + username=connection_info["username"], + password=connection_info["password"], + pkey=connection_info["pkey"], + ) + try: + # "exit 0" should work across platforms. + # "whoami" would also work. + utils.exec_ssh_cmd(ssh, "exit 0") + finally: + ssh.close() + LOG.debug("Instance reachable: %s", connection_info["ip"]) + return + except Exception as err: + LOG.debug( + f"Could not connect to remote instance: {str(err)}. " + f"Retrying, time left: {timeout - (time.time() - start)}." + ) + time.sleep(poll_interval) + + raise exception.CoriolisException( + f"Operation timed out after waiting {timeout}s for the instance to be " + f"accessible via SSH." + ) + + +def _poll_instance_until_reachable_winrm( + connection_info: dict, + timeout: int = 600, + poll_interval: int = 10, +): + start = time.time() + while (time.time() - start) < timeout: + try: + conn = wsman.WSManConnection.from_connection_info(connection_info) + conn.exec_ps_command("whoami") + return + except Exception as ex: + LOG.debug( + f"Could not conect to Windows host: {str(ex)}. " + f"Retrying, time left: {timeout - (time.time() - start)}." + ) + time.sleep(poll_interval) + + raise exception.CoriolisException( + f"Operation timed out after waiting {timeout}s for Windows host to " + f"be accessible via WinRM." + ) + + +def poll_instance_until_reachable( + connection_info: dict, + protocol: str = constants.PROTOCOL_SSH, + timeout: int = 600, + poll_interval: int = 10, +) -> paramiko.SSHClient: + """Poll until a given instance becomes reachable. + + :param connection_info: a dict containing the following keys: + * ip + * port + * username + * password + * pkey - Paramiko keypair + :param protocol: connection protocol, "ssh" or "winrm" + :param timeout: the maximum amount of time to wait + :param poll_interval: the amount of time to wait between retries + """ + # TODO(lpetrut): consider including the connection protocol in the + # connection info. We'd have to modify a few schemas used during os + # morphing. We currently pick the protocol based on the OS type but + # we may want to use SSH on Windows as well. + if protocol == constants.PROTOCOL_SSH: + helper = _poll_instance_until_reachable_ssh + elif protocol == constants.PROTOCOL_WINRM: + helper = _poll_instance_until_reachable_winrm + else: + raise exception.InvalidInput( + f"Unsupported instance connection protocol: {protocol}") + return helper( + connection_info=connection_info, + timeout=timeout, + poll_interval=poll_interval) diff --git a/coriolis/tests/providers/test_backup_writers.py b/coriolis/tests/providers/test_backup_writers.py index 4492336a..d0ab766f 100644 --- a/coriolis/tests/providers/test_backup_writers.py +++ b/coriolis/tests/providers/test_backup_writers.py @@ -164,6 +164,49 @@ def test__validate_info_with_missing_connection_details(self): self.assertRaises(exception.CoriolisException, self._get_factory, {"backend": "ssh"}) + @mock.patch.object(backup_writers, "HTTPBackupWriterBootstrapper") + def test_get_conn_info_https(self, mock_https_bootstrapper): + mechanism = backup_writers.DATA_TRANSFER_MECHANISM_HTTPS + factory = backup_writers.BackupWritersFactory + writer_conn_info = factory.get_backup_writer_connection_info( + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism=mechanism, + ) + self.assertEqual( + writer_conn_info["backend"], backup_writers.BACKUP_WRITER_HTTP) + self.assertEqual( + writer_conn_info["connection_details"], + mock_https_bootstrapper.return_value.setup_writer.return_value) + mock_https_bootstrapper.assert_called_once_with( + mock.sentinel.instance_conn_info, + backup_writers.DATA_TRANSFER_MECHANISM_HTTPS_PORT, + ) + + def test_get_conn_info_ssh(self): + mechanism = backup_writers.DATA_TRANSFER_MECHANISM_SSH + factory = backup_writers.BackupWritersFactory + writer_conn_info = factory.get_backup_writer_connection_info( + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism=mechanism, + ) + self.assertEqual( + writer_conn_info["backend"], backup_writers.BACKUP_WRITER_SSH) + self.assertEqual( + writer_conn_info["connection_details"], + mock.sentinel.instance_conn_info) + + def test_get_conn_info_unsupported(self): + factory = backup_writers.BackupWritersFactory + self.assertRaises( + ValueError, + factory.get_backup_writer_connection_info, + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism="fake-mechanism", + ) + class BaseBackupWriterTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis BaseBackupWriter class.""" diff --git a/coriolis/tests/providers/test_provider_utils.py b/coriolis/tests/providers/test_provider_utils.py index d77dee2b..c66ac7ce 100644 --- a/coriolis/tests/providers/test_provider_utils.py +++ b/coriolis/tests/providers/test_provider_utils.py @@ -2,15 +2,27 @@ # All Rights Reserved. import logging +from unittest import mock +from coriolis import constants from coriolis import exception from coriolis.providers import provider_utils from coriolis.tests import test_base +from coriolis import utils class ProviderUtilsTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis provider utils module.""" + def _get_mock_conn_info(self): + return { + "ip": "1.2.3.4", + "port": 2222, + "username": "someUser", + "password": "pwned", + "pkey": mock.Mock(), + } + def test_get_storage_mapping_for_disk(self): storage_mappings = { 'disk_mappings': [{'disk_id': '1', 'destination': 'backend1'}], @@ -257,3 +269,122 @@ def test_check_changed_storage_mappings_empty_new_mappings(self): provider_utils.check_changed_storage_mappings, volumes_info, old_storage_mappings, new_storage_mappings) + + @mock.patch.object(utils, "connect_ssh") + @mock.patch.object(utils, "exec_ssh_cmd") + @mock.patch("time.sleep") + def test_poll_instance_ssh( + self, + mock_sleep, + mock_exec_ssh, + mock_connect, + ): + mock_ssh_conn = mock.Mock() + mock_connect.side_effect = [ + Exception, + mock_ssh_conn, + mock_ssh_conn, + ] + mock_exec_ssh.side_effect = [ + Exception, + mock.sentinel.stdout, + ] + poll_interval = 5 + + connection_info = self._get_mock_conn_info() + provider_utils.poll_instance_until_reachable( + connection_info=connection_info, + protocol=constants.PROTOCOL_SSH, + timeout=30, + poll_interval=poll_interval, + ) + + mock_connect.assert_has_calls( + [ + mock.call( + hostname=connection_info["ip"], + port=connection_info["port"], + username=connection_info["username"], + password=connection_info["password"], + pkey=connection_info["pkey"], + ) + ] * 2) + mock_exec_ssh.assert_has_calls( + [mock.call(mock_ssh_conn, "exit 0")] * 2) + mock_ssh_conn.close.assert_has_calls([mock.call()] * 2) + mock_sleep.assert_has_calls([mock.call(poll_interval)] * 2) + + @mock.patch.object(utils, "connect_ssh") + @mock.patch.object(utils, "exec_ssh_cmd") + @mock.patch("time.sleep") + @mock.patch("time.time") + def test_poll_instance_ssh_timeout( + self, + mock_time, + mock_sleep, + mock_exec_ssh, + mock_connect, + ): + poll_interval = 5 + mock_time.side_effect = [x * 10 for x in range(20)] + mock_connect.side_effect = IOError + connection_info = self._get_mock_conn_info() + self.assertRaises( + exception.CoriolisException, + provider_utils.poll_instance_until_reachable, + connection_info=connection_info, + protocol=constants.PROTOCOL_SSH, + timeout=30, + poll_interval=poll_interval, + ) + + @mock.patch("coriolis.wsman.WSManConnection", new_callable=mock.Mock) + @mock.patch("time.sleep") + def test_poll_instance_winrm( + self, + mock_sleep, + mock_wsman, + ): + mock_conn = mock.Mock() + mock_conn.exec_ps_command.side_effect = [ + Exception, mock.sentinel.stdout] + mock_wsman.from_connection_info.return_value = mock_conn + poll_interval = 5 + + connection_info = self._get_mock_conn_info() + provider_utils.poll_instance_until_reachable( + connection_info=connection_info, + protocol=constants.PROTOCOL_WINRM, + timeout=30, + poll_interval=poll_interval, + ) + + mock_wsman.from_connection_info.assert_has_calls( + [mock.call(connection_info)] * 2, any_order=True) + mock_conn.exec_ps_command.assert_has_calls( + [mock.call("whoami")] * 2) + mock_sleep.assert_called_once_with(poll_interval) + + @mock.patch("coriolis.wsman.WSManConnection", new_callable=mock.Mock) + @mock.patch("time.sleep") + @mock.patch("time.time") + def test_poll_instance_winrm_timeout( + self, + mock_time, + mock_sleep, + mock_wsman, + ): + poll_interval = 5 + mock_time.side_effect = [x * 10 for x in range(20)] + mock_conn = mock.Mock() + mock_conn.exec_ps_command.side_effect = IOError + mock_wsman.from_connection_info.return_value = mock_conn + connection_info = self._get_mock_conn_info() + self.assertRaises( + exception.CoriolisException, + provider_utils.poll_instance_until_reachable, + connection_info=connection_info, + protocol=constants.PROTOCOL_WINRM, + timeout=30, + poll_interval=poll_interval, + )