From 947a316a784580da0e91685b44635067e82e0038 Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Wed, 26 Aug 2020 12:22:46 -0300 Subject: Fix cloud config on chef example (#551) --- doc/examples/cloud-config-chef.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index bb4b058c..8cebfd80 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -13,7 +13,8 @@ # Key from https://packages.chef.io/chef.asc apt: sources: - source1: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" + source1: + source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" key: | -----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1.4.12 (Darwin) -- cgit v1.2.3 From 371b392ced518e45be51089b6a67b362957b1dba Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 26 Aug 2020 09:58:28 -0700 Subject: util: remove debug statement (#556) --- cloudinit/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index cf9e349f..64142f23 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2490,7 +2490,6 @@ def read_dmi_data(key): LOG.debug("dmidata is not supported on %s", uname_arch) return None - print("hi, now its: %s\n", subp) dmidecode_path = subp.which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) -- cgit v1.2.3 From 575750ac4a3abb6fdfa45e2cc65186df291d8ddf Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 27 Aug 2020 16:37:14 +0200 Subject: redhat spec: add missing BuildRequires (#552) 456fb55744a1acc6bd2f464b7656a9c33d0b7ac5 made tools/read-dependencies and package/brpm distinguish between build dependencies and runtime dependencies, however packages/redhat/cloud-init.spec.in expects all the dependencies to be in the 'requires' list, thus missing some build dependencies. This change makes cloud-init.spec use 'buildrequires' too. The build happens to succeed without python3-devel on the epel-8 copr chroot as it pulls in the epel-rpm-macros package, which in turn depends on python3-devel. In other words the dependency is satisfied by chance. Packages building for Python 3 need to explicitly specify BuildRequires: python3-devel, see: [1]. [1] https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/ --- packages/redhat/cloud-init.spec.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 4cff2c97..16138012 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -39,10 +39,7 @@ Requires(post): chkconfig Requires(preun): chkconfig %endif -# These are runtime dependencies, but declared as BuildRequires so that -# - tests can be run here. -# - parts of cloud-init such (setup.py) use these dependencies. -{% for r in requires %} +{% for r in buildrequires %} BuildRequires: {{r}} {% endfor %} @@ -57,8 +54,10 @@ Requires: python-argparse %endif -# Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json +# Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json. +# Install them as BuildRequires too as they're used for testing. {% for r in requires %} +BuildRequires: {{r}} Requires: {{r}} {% endfor %} -- cgit v1.2.3 From 1f3a225af78dbfbff75c3faad28a5dc8cad0d1e3 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 27 Aug 2020 17:20:35 +0200 Subject: LXD: detach network from profile before deleting it (#542) * LXD: detach network from profile before deleting it When cleaning up the bridge network created by default by LXD as part of the `lxd init` process detach the network its profile before deleting it. LXD will otherwise refuse to delete it with error: Error: The network is currently in use. Discussion with LXD upstream: https://github.com/lxc/lxd/issues/7804. LP: #1776958 * LXD bridge deletion: fail if bridge exists but can't be deleted * LXD bridge deletion: remove useless failure logging --- cloudinit/config/cc_lxd.py | 12 ++++++++---- tests/unittests/test_handler/test_handler_lxd.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7129c9c6..486037d9 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -283,14 +283,18 @@ def maybe_cleanup_default(net_name, did_init, create, attach, fail_assume_enoent = "failed. Assuming it did not exist." succeeded = "succeeded." if create: - msg = "Deletion of lxd network '%s' %s" + msg = "Detach of lxd network '%s' from profile '%s' %s" try: - _lxc(["network", "delete", net_name]) - LOG.debug(msg, net_name, succeeded) + _lxc(["network", "detach-profile", net_name, profile]) + LOG.debug(msg, net_name, profile, succeeded) except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e - LOG.debug(msg, net_name, fail_assume_enoent) + LOG.debug(msg, net_name, profile, fail_assume_enoent) + else: + msg = "Deletion of lxd network '%s' %s" + _lxc(["network", "delete", net_name]) + LOG.debug(msg, net_name, succeeded) if attach: msg = "Removal of device '%s' from profile '%s' %s" diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 21011204..b2181992 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -214,7 +214,7 @@ class TestLxdMaybeCleanupDefault(t_help.CiTestCase): """deletion of network should occur if create is True.""" cc_lxd.maybe_cleanup_default( net_name=self.defnet, did_init=True, create=True, attach=False) - m_lxc.assert_called_once_with(["network", "delete", self.defnet]) + m_lxc.assert_called_with(["network", "delete", self.defnet]) @mock.patch("cloudinit.config.cc_lxd._lxc") def test_device_removed_if_attach_true(self, m_lxc): -- cgit v1.2.3 From 20fe9da98ed69187faab80810c652fdc051b627e Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 27 Aug 2020 19:16:13 +0200 Subject: systemd: add Before=shutdown.target when Conflicts=shutdown.target is used (#546) Lintian spotted the following issue: The specified systemd .service file contains both DefaultDependencies=no and Conflicts=shutdown.target directives without Before=shutdown.target. This can lead to problems during shutdown because the service may linger until the very end of shutdown sequence as nothing requests to stop it before (due to DefaultDependencies=no). There is race condition between stopping units and systemd getting a request to exit the main loop, so it may proceed with shutdown before all pending stop jobs have been processed. Please add Before=shutdown.target. [1] https://lintian.debian.org/tags/systemd-service-file-shutdown-problems.html Co-authored-by: Rick Harding --- systemd/cloud-init.service.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index af6d9a8b..f140344d 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -25,9 +25,11 @@ Before=sshd-keygen.service Before=sshd.service {% if variant in ["ubuntu", "unknown", "debian"] %} Before=sysinit.target +Before=shutdown.target Conflicts=shutdown.target {% endif %} {% if variant in ["suse"] %} +Before=shutdown.target Conflicts=shutdown.target {% endif %} Before=systemd-user-sessions.service -- cgit v1.2.3 From 13362f536e9d8a092ec20dcb5abe7a0b86407f45 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Fri, 28 Aug 2020 08:23:59 -0700 Subject: Add method type hints for Azure helper (#540) This reverts commit 8d25d5e6fac39ab3319ec5d37d23196429fb0c95. --- cloudinit/sources/helpers/azure.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index b968a96f..507f6ac8 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -288,7 +288,8 @@ class InvalidGoalStateXMLException(Exception): class GoalState: - def __init__(self, unparsed_xml, azure_endpoint_client): + def __init__(self, unparsed_xml: str, + azure_endpoint_client: AzureEndpointHttpClient) -> None: """Parses a GoalState XML string and returns a GoalState object. @param unparsed_xml: string representing a GoalState XML. @@ -478,7 +479,10 @@ class GoalStateHealthReporter: PROVISIONING_SUCCESS_STATUS = 'Ready' - def __init__(self, goal_state, azure_endpoint_client, endpoint): + def __init__( + self, goal_state: GoalState, + azure_endpoint_client: AzureEndpointHttpClient, + endpoint: str) -> None: """Creates instance that will report provisioning status to an endpoint @param goal_state: An instance of class GoalState that contains @@ -495,7 +499,7 @@ class GoalStateHealthReporter: self._endpoint = endpoint @azure_ds_telemetry_reporter - def send_ready_signal(self): + def send_ready_signal(self) -> None: document = self.build_report( incarnation=self._goal_state.incarnation, container_id=self._goal_state.container_id, @@ -513,8 +517,8 @@ class GoalStateHealthReporter: LOG.info('Reported ready to Azure fabric.') def build_report( - self, incarnation, container_id, instance_id, - status, substatus=None, description=None): + self, incarnation: str, container_id: str, instance_id: str, + status: str, substatus=None, description=None) -> str: health_detail = '' if substatus is not None: health_detail = self.HEALTH_DETAIL_SUBSECTION_XML_TEMPLATE.format( @@ -530,7 +534,7 @@ class GoalStateHealthReporter: return health_report @azure_ds_telemetry_reporter - def _post_health_report(self, document): + def _post_health_report(self, document: str) -> None: push_log_to_kvp() # Whenever report_diagnostic_event(diagnostic_msg) is invoked in code, @@ -726,7 +730,7 @@ class WALinuxAgentShim: return endpoint_ip_address @azure_ds_telemetry_reporter - def register_with_azure_and_fetch_data(self, pubkey_info=None): + def register_with_azure_and_fetch_data(self, pubkey_info=None) -> dict: """Gets the VM's GoalState from Azure, uses the GoalState information to report ready/send the ready signal/provisioning complete signal to Azure, and then uses pubkey_info to filter and obtain the user's @@ -750,7 +754,7 @@ class WALinuxAgentShim: return {'public-keys': ssh_keys} @azure_ds_telemetry_reporter - def _fetch_goal_state_from_azure(self): + def _fetch_goal_state_from_azure(self) -> GoalState: """Fetches the GoalState XML from the Azure endpoint, parses the XML, and returns a GoalState object. @@ -760,7 +764,7 @@ class WALinuxAgentShim: return self._parse_raw_goal_state_xml(unparsed_goal_state_xml) @azure_ds_telemetry_reporter - def _get_raw_goal_state_xml_from_azure(self): + def _get_raw_goal_state_xml_from_azure(self) -> str: """Fetches the GoalState XML from the Azure endpoint and returns the XML as a string. @@ -780,7 +784,8 @@ class WALinuxAgentShim: return response.contents @azure_ds_telemetry_reporter - def _parse_raw_goal_state_xml(self, unparsed_goal_state_xml): + def _parse_raw_goal_state_xml( + self, unparsed_goal_state_xml: str) -> GoalState: """Parses a GoalState XML string and returns a GoalState object. @param unparsed_goal_state_xml: GoalState XML string @@ -803,7 +808,8 @@ class WALinuxAgentShim: return goal_state @azure_ds_telemetry_reporter - def _get_user_pubkeys(self, goal_state, pubkey_info): + def _get_user_pubkeys( + self, goal_state: GoalState, pubkey_info: list) -> list: """Gets and filters the VM admin user's authorized pubkeys. The admin user in this case is the username specified as "admin" @@ -838,7 +844,7 @@ class WALinuxAgentShim: return ssh_keys @staticmethod - def _filter_pubkeys(keys_by_fingerprint, pubkey_info): + def _filter_pubkeys(keys_by_fingerprint: dict, pubkey_info: list) -> list: """ Filter and return only the user's actual pubkeys. @param keys_by_fingerprint: pubkey fingerprint -> pubkey value dict -- cgit v1.2.3 From a802a1243f3bcd72617b6969e5134752666960ca Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Mon, 31 Aug 2020 11:23:59 -0400 Subject: Run cloud-init-local.service after the hv_kvp_daemon (#505) This ensures that KVP data can be correctly reported to Azure. --- systemd/cloud-init-local.service.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index ff9c644d..7166f640 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -5,6 +5,7 @@ Description=Initial cloud-init job (pre-networking) DefaultDependencies=no {% endif %} Wants=network-pre.target +After=hv_kvp_daemon.service After=systemd-remount-fs.service Before=NetworkManager.service Before=network-pre.target -- cgit v1.2.3 From 9d7d54dc42a272cc2b506a310d59281cd40e5210 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Tue, 1 Sep 2020 21:57:18 +0200 Subject: network-config-format-v1: fix typo in nameserver example (#564) --- doc/rtd/topics/network-config-format-v1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rtd/topics/network-config-format-v1.rst b/doc/rtd/topics/network-config-format-v1.rst index 9723d689..dfbde514 100644 --- a/doc/rtd/topics/network-config-format-v1.rst +++ b/doc/rtd/topics/network-config-format-v1.rst @@ -332,7 +332,7 @@ the following keys: - type: static address: 192.168.23.14/27 gateway: 192.168.23.1 - - type: nameserver: + - type: nameserver address: - 192.168.23.2 - 8.8.8.8 -- cgit v1.2.3 From 86748592b90035967525b3ee6b199a377aaf2ee0 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 3 Sep 2020 14:59:41 +0200 Subject: Bump the integration-requirements versioned dependencies (#565) During the ec2 integration test runs we occasionally see failures in deleting test instances. Hopefully a newer boto3 will be more robust. Also bump: paramiko, cryptography, pylxd (now pulling it from pypi). Tested with a full Xenial EC2 cloud_tests run. --- integration-requirements.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/integration-requirements.txt b/integration-requirements.txt index 44e45c1b..13cfb9d7 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -6,16 +6,14 @@ # # ec2 backend -boto3==1.5.9 +boto3==1.14.53 # ssh communication -paramiko==2.4.2 -cryptography==2.4.2 - +paramiko==2.7.2 +cryptography==3.1 # lxd backend -# 04/03/2018: enables use of lxd 3.0 -git+https://github.com/lxc/pylxd.git@4b8ab1802f9aee4eb29cf7b119dae0aa47150779 +pylxd==2.2.11 # finds latest image information git+https://git.launchpad.net/simplestreams -- cgit v1.2.3 From 2a95dfb52390adadd3b9e733717356cb37c8d892 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 8 Sep 2020 14:06:45 -0400 Subject: distros: minor typo fix (#562) Co-authored-by: Rick Harding --- cloudinit/distros/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2537608f..2bd9bae8 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -250,8 +250,9 @@ class Distro(metaclass=abc.ABCMeta): distros = [] for family in family_list: if family not in OSFAMILIES: - raise ValueError("No distibutions found for osfamily %s" - % (family)) + raise ValueError( + "No distributions found for osfamily {}".format(family) + ) distros.extend(OSFAMILIES[family]) return distros -- cgit v1.2.3 From 987f89efa1615087c6d87470f758fc609b14e1b8 Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Tue, 8 Sep 2020 19:39:18 +0000 Subject: Add jqueuniet as contributor (#569) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c67db436..aa946511 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -8,6 +8,7 @@ dhensby eandersson izzyleung johnsonshi +jqueuniet landon912 lucasmoura marlluslustosa -- cgit v1.2.3 From e56b55452549cb037da0a4165154ffa494e9678a Mon Sep 17 00:00:00 2001 From: Thomas Stringer Date: Thu, 10 Sep 2020 14:29:54 -0400 Subject: Retrieve SSH keys from IMDS first with OVF as a fallback (#509) * pull ssh keys from imds first and fall back to ovf if unavailable * refactor log and diagnostic messages * refactor the OpenSSLManager instantiation and certificate usage * fix unit test where exception was being silenced for generate cert * fix tests now that certificate is not always generated * add documentation for ssh key retrieval * add ability to check if http client has security enabled * refactor certificate logic to GoalState --- cloudinit/sources/DataSourceAzure.py | 53 +++++++++++++++++- cloudinit/sources/helpers/azure.py | 50 ++++++++++++----- doc/rtd/topics/datasources/azure.rst | 6 ++ tests/unittests/test_datasource/test_azure.py | 64 ++++++++++++++++++---- .../unittests/test_datasource/test_azure_helper.py | 13 +++-- 5 files changed, 156 insertions(+), 30 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f3c6452b..e98fd497 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -561,6 +561,40 @@ class DataSourceAzure(sources.DataSource): def device_name_to_device(self, name): return self.ds_cfg['disk_aliases'].get(name) + @azure_ds_telemetry_reporter + def get_public_ssh_keys(self): + """ + Try to get the ssh keys from IMDS first, and if that fails + (i.e. IMDS is unavailable) then fallback to getting the ssh + keys from OVF. + + The benefit to getting keys from IMDS is a large performance + advantage, so this is a strong preference. But we must keep + OVF as a second option for environments that don't have IMDS. + """ + LOG.debug('Retrieving public SSH keys') + ssh_keys = [] + try: + ssh_keys = [ + public_key['keyData'] + for public_key + in self.metadata['imds']['compute']['publicKeys'] + ] + LOG.debug('Retrieved SSH keys from IMDS') + except KeyError: + log_msg = 'Unable to get keys from IMDS, falling back to OVF' + LOG.debug(log_msg) + report_diagnostic_event(log_msg) + try: + ssh_keys = self.metadata['public-keys'] + LOG.debug('Retrieved keys from OVF') + except KeyError: + log_msg = 'No keys available from OVF' + LOG.debug(log_msg) + report_diagnostic_event(log_msg) + + return ssh_keys + def get_config_obj(self): return self.cfg @@ -764,7 +798,22 @@ class DataSourceAzure(sources.DataSource): if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN: self.bounce_network_with_azure_hostname() - pubkey_info = self.cfg.get('_pubkeys', None) + pubkey_info = None + try: + public_keys = self.metadata['imds']['compute']['publicKeys'] + LOG.debug( + 'Successfully retrieved %s key(s) from IMDS', + len(public_keys) + if public_keys is not None + else 0 + ) + except KeyError: + LOG.debug( + 'Unable to retrieve SSH keys from IMDS during ' + 'negotiation, falling back to OVF' + ) + pubkey_info = self.cfg.get('_pubkeys', None) + metadata_func = partial(get_metadata_from_fabric, fallback_lease_file=self. dhclient_lease_file, @@ -1443,7 +1492,7 @@ def get_metadata_from_imds(fallback_nic, retries): @azure_ds_telemetry_reporter def _get_metadata_from_imds(retries): - url = IMDS_URL + "instance?api-version=2017-12-01" + url = IMDS_URL + "instance?api-version=2019-06-01" headers = {"Metadata": "true"} try: response = readurl( diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 507f6ac8..79445a81 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -288,12 +288,16 @@ class InvalidGoalStateXMLException(Exception): class GoalState: - def __init__(self, unparsed_xml: str, - azure_endpoint_client: AzureEndpointHttpClient) -> None: + def __init__( + self, + unparsed_xml: str, + azure_endpoint_client: AzureEndpointHttpClient, + need_certificate: bool = True) -> None: """Parses a GoalState XML string and returns a GoalState object. @param unparsed_xml: string representing a GoalState XML. - @param azure_endpoint_client: instance of AzureEndpointHttpClient + @param azure_endpoint_client: instance of AzureEndpointHttpClient. + @param need_certificate: switch to know if certificates is needed. @return: GoalState object representing the GoalState XML string. """ self.azure_endpoint_client = azure_endpoint_client @@ -322,7 +326,7 @@ class GoalState: url = self._text_from_xpath( './Container/RoleInstanceList/RoleInstance' '/Configuration/Certificates') - if url is not None: + if url is not None and need_certificate: with events.ReportEventStack( name="get-certificates-xml", description="get certificates xml", @@ -741,27 +745,38 @@ class WALinuxAgentShim: GoalState. @return: The list of user's authorized pubkey values. """ - if self.openssl_manager is None: + http_client_certificate = None + if self.openssl_manager is None and pubkey_info is not None: self.openssl_manager = OpenSSLManager() + http_client_certificate = self.openssl_manager.certificate if self.azure_endpoint_client is None: self.azure_endpoint_client = AzureEndpointHttpClient( - self.openssl_manager.certificate) - goal_state = self._fetch_goal_state_from_azure() - ssh_keys = self._get_user_pubkeys(goal_state, pubkey_info) + http_client_certificate) + goal_state = self._fetch_goal_state_from_azure( + need_certificate=http_client_certificate is not None + ) + ssh_keys = None + if pubkey_info is not None: + ssh_keys = self._get_user_pubkeys(goal_state, pubkey_info) health_reporter = GoalStateHealthReporter( goal_state, self.azure_endpoint_client, self.endpoint) health_reporter.send_ready_signal() return {'public-keys': ssh_keys} @azure_ds_telemetry_reporter - def _fetch_goal_state_from_azure(self) -> GoalState: + def _fetch_goal_state_from_azure( + self, + need_certificate: bool) -> GoalState: """Fetches the GoalState XML from the Azure endpoint, parses the XML, and returns a GoalState object. @return: GoalState object representing the GoalState XML """ unparsed_goal_state_xml = self._get_raw_goal_state_xml_from_azure() - return self._parse_raw_goal_state_xml(unparsed_goal_state_xml) + return self._parse_raw_goal_state_xml( + unparsed_goal_state_xml, + need_certificate + ) @azure_ds_telemetry_reporter def _get_raw_goal_state_xml_from_azure(self) -> str: @@ -774,7 +789,11 @@ class WALinuxAgentShim: LOG.info('Registering with Azure...') url = 'http://{}/machine/?comp=goalstate'.format(self.endpoint) try: - response = self.azure_endpoint_client.get(url) + with events.ReportEventStack( + name="goalstate-retrieval", + description="retrieve goalstate", + parent=azure_ds_reporter): + response = self.azure_endpoint_client.get(url) except Exception as e: msg = 'failed to register with Azure: %s' % e LOG.warning(msg) @@ -785,7 +804,9 @@ class WALinuxAgentShim: @azure_ds_telemetry_reporter def _parse_raw_goal_state_xml( - self, unparsed_goal_state_xml: str) -> GoalState: + self, + unparsed_goal_state_xml: str, + need_certificate: bool) -> GoalState: """Parses a GoalState XML string and returns a GoalState object. @param unparsed_goal_state_xml: GoalState XML string @@ -793,7 +814,10 @@ class WALinuxAgentShim: """ try: goal_state = GoalState( - unparsed_goal_state_xml, self.azure_endpoint_client) + unparsed_goal_state_xml, + self.azure_endpoint_client, + need_certificate + ) except Exception as e: msg = 'Error processing GoalState XML: %s' % e LOG.warning(msg) diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index fdb919a5..e04c3a33 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -68,6 +68,12 @@ configuration information to the instance. Cloud-init uses the IMDS for: - network configuration for the instance which is applied per boot - a preprovisioing gate which blocks instance configuration until Azure fabric is ready to provision +- retrieving SSH public keys. Cloud-init will first try to utilize SSH keys + returned from IMDS, and if they are not provided from IMDS then it will + fallback to using the OVF file provided from the CD-ROM. There is a large + performance benefit to using IMDS for SSH key retrieval, but in order to + support environments where IMDS is not available then we must continue to + all for keys from OVF Configuration diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 47e03bd1..2dda9925 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -102,7 +102,13 @@ NETWORK_METADATA = { "vmId": "ff702a6b-cb6a-4fcd-ad68-b4ce38227642", "vmScaleSetName": "", "vmSize": "Standard_DS1_v2", - "zone": "" + "zone": "", + "publicKeys": [ + { + "keyData": "key1", + "path": "path1" + } + ] }, "network": { "interface": [ @@ -302,7 +308,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): def setUp(self): super(TestGetMetadataFromIMDS, self).setUp() - self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2017-12-01" + self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01" @mock.patch(MOCKPATH + 'readurl') @mock.patch(MOCKPATH + 'EphemeralDHCPv4') @@ -1304,6 +1310,40 @@ scbus-1 on xpt0 bus 0 dsaz.get_hostname(hostname_command=("hostname",)) m_subp.assert_called_once_with(("hostname",), capture=True) + @mock.patch( + 'cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates') + def test_get_public_ssh_keys_with_imds(self, m_parse_certificates): + sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = { + 'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': sys_cfg + } + dsrc = self._get_ds(data) + dsrc.get_data() + dsrc.setup(True) + ssh_keys = dsrc.get_public_ssh_keys() + self.assertEqual(ssh_keys, ['key1']) + self.assertEqual(m_parse_certificates.call_count, 0) + + @mock.patch(MOCKPATH + 'get_metadata_from_imds') + def test_get_public_ssh_keys_without_imds( + self, + m_get_metadata_from_imds): + m_get_metadata_from_imds.return_value = dict() + sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = { + 'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': sys_cfg + } + dsrc = self._get_ds(data) + dsaz.get_metadata_from_fabric.return_value = {'public-keys': ['key2']} + dsrc.get_data() + dsrc.setup(True) + ssh_keys = dsrc.get_public_ssh_keys() + self.assertEqual(ssh_keys, ['key2']) + class TestAzureBounce(CiTestCase): @@ -2094,14 +2134,18 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): md, _ud, cfg, _d = dsa._reprovision() self.assertEqual(md['local-hostname'], hostname) self.assertEqual(cfg['system_info']['default_user']['name'], username) - self.assertEqual(fake_resp.call_args_list, - [mock.call(allow_redirects=True, - headers={'Metadata': 'true', - 'User-Agent': - 'Cloud-Init/%s' % vs()}, - method='GET', - timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, - url=full_url)]) + self.assertIn( + mock.call( + allow_redirects=True, + headers={ + 'Metadata': 'true', + 'User-Agent': 'Cloud-Init/%s' % vs() + }, + method='GET', + timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, + url=full_url + ), + fake_resp.call_args_list) self.assertEqual(m_dhcp.call_count, 2) m_net.assert_any_call( broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5e6d3d2d..5c31b8be 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -609,11 +609,11 @@ class TestWALinuxAgentShim(CiTestCase): self.GoalState.return_value.container_id = self.test_container_id self.GoalState.return_value.instance_id = self.test_instance_id - def test_azure_endpoint_client_uses_certificate_during_report_ready(self): + def test_http_client_does_not_use_certificate(self): shim = wa_shim() shim.register_with_azure_and_fetch_data() self.assertEqual( - [mock.call(self.OpenSSLManager.return_value.certificate)], + [mock.call(None)], self.AzureEndpointHttpClient.call_args_list) def test_correct_url_used_for_goalstate_during_report_ready(self): @@ -625,8 +625,11 @@ class TestWALinuxAgentShim(CiTestCase): [mock.call('http://test_endpoint/machine/?comp=goalstate')], get.call_args_list) self.assertEqual( - [mock.call(get.return_value.contents, - self.AzureEndpointHttpClient.return_value)], + [mock.call( + get.return_value.contents, + self.AzureEndpointHttpClient.return_value, + False + )], self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): @@ -701,7 +704,7 @@ class TestWALinuxAgentShim(CiTestCase): shim.register_with_azure_and_fetch_data() shim.clean_up() self.assertEqual( - 1, self.OpenSSLManager.return_value.clean_up.call_count) + 0, self.OpenSSLManager.return_value.clean_up.call_count) def test_fetch_goalstate_during_report_ready_raises_exc_on_get_exc(self): self.AzureEndpointHttpClient.return_value.get \ -- cgit v1.2.3 From 8439b191ec2f336d544cab86dba2860f969cd5b8 Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Tue, 15 Sep 2020 18:00:00 +0200 Subject: network: Fix type and respect name when rendering vlan in sysconfig. (#541) Prior to this change, vlans were rendered in sysconfig with 'TYPE=Ethernet', and incorrectly rendered the PHYSDEV based on the name of the vlan device rather than the 'link' provided in the network config. The change here fixes: * rendering of TYPE=Ethernet for a vlan * adds a warning if the configured device name is not supported per the RHEL 7 docs "11.5. Naming Scheme for VLAN Interfaces" LP: #1788915 LP: #1826608 RHBZ: #1861871 --- cloudinit/net/sysconfig.py | 32 +++++++++- tests/unittests/test_distros/test_netconfig.py | 81 ++++++++++++++++++++++++++ tests/unittests/test_net.py | 4 -- 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 0a5d481d..e9337b12 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -99,6 +99,10 @@ class ConfigMap(object): def __len__(self): return len(self._conf) + def skip_key_value(self, key, val): + """Skip the pair key, value if it matches a certain rule.""" + return False + def to_string(self): buf = io.StringIO() buf.write(_make_header()) @@ -106,6 +110,8 @@ class ConfigMap(object): buf.write("\n") for key in sorted(self._conf.keys()): value = self._conf[key] + if self.skip_key_value(key, value): + continue if isinstance(value, bool): value = self._bool_map[value] if not isinstance(value, str): @@ -214,6 +220,7 @@ class NetInterface(ConfigMap): 'bond': 'Bond', 'bridge': 'Bridge', 'infiniband': 'InfiniBand', + 'vlan': 'Vlan', } def __init__(self, iface_name, base_sysconf_dir, templates, @@ -267,6 +274,11 @@ class NetInterface(ConfigMap): c.routes = self.routes.copy() return c + def skip_key_value(self, key, val): + if key == 'TYPE' and val == 'Vlan': + return True + return False + class Renderer(renderer.Renderer): """Renders network information in a /etc/sysconfig format.""" @@ -697,7 +709,16 @@ class Renderer(renderer.Renderer): iface_cfg['ETHERDEVICE'] = iface_name[:iface_name.rfind('.')] else: iface_cfg['VLAN'] = True - iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + iface_cfg.kind = 'vlan' + + rdev = iface['vlan-raw-device'] + supported = _supported_vlan_names(rdev, iface['vlan_id']) + if iface_name not in supported: + LOG.info( + "Name '%s' for vlan '%s' is not officially supported" + "by RHEL. Supported: %s", + iface_name, rdev, ' '.join(supported)) + iface_cfg['PHYSDEV'] = rdev iface_subnets = iface.get("subnets", []) route_cfg = iface_cfg.routes @@ -896,6 +917,15 @@ class Renderer(renderer.Renderer): "\n".join(netcfg) + "\n", file_mode) +def _supported_vlan_names(rdev, vid): + """Return list of supported names for vlan devices per RHEL doc + 11.5. Naming Scheme for VLAN Interfaces.""" + return [ + v.format(rdev=rdev, vid=int(vid)) + for v in ("{rdev}{vid:04}", "{rdev}{vid}", + "{rdev}.{vid:04}", "{rdev}.{vid}")] + + def available(target=None): sysconfig = available_sysconfig(target=target) nm = available_nm(target=target) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 8d7b09c8..3f3fe3eb 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -539,6 +539,87 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase): V1_NET_CFG_IPV6, expected_cfgs=expected_cfgs.copy()) + def test_vlan_render_unsupported(self): + """Render officially unsupported vlan names.""" + cfg = { + 'version': 2, + 'ethernets': { + 'eth0': {'addresses': ["192.10.1.2/24"], + 'match': {'macaddress': "00:16:3e:60:7c:df"}}}, + 'vlans': { + 'infra0': {'addresses': ["10.0.1.2/16"], + 'id': 1001, 'link': 'eth0'}}, + } + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEVICE=eth0 + HWADDR=00:16:3e:60:7c:df + IPADDR=192.10.1.2 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('infra0'): dedent("""\ + BOOTPROTO=none + DEVICE=infra0 + IPADDR=10.0.1.2 + NETMASK=255.255.0.0 + NM_CONTROLLED=no + ONBOOT=yes + PHYSDEV=eth0 + USERCTL=no + VLAN=yes + """), + self.control_path(): dedent("""\ + NETWORKING=yes + """), + } + self._apply_and_verify( + self.distro.apply_network_config, cfg, + expected_cfgs=expected_cfgs) + + def test_vlan_render(self): + cfg = { + 'version': 2, + 'ethernets': { + 'eth0': {'addresses': ["192.10.1.2/24"]}}, + 'vlans': { + 'eth0.1001': {'addresses': ["10.0.1.2/16"], + 'id': 1001, 'link': 'eth0'}}, + } + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEVICE=eth0 + IPADDR=192.10.1.2 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('eth0.1001'): dedent("""\ + BOOTPROTO=none + DEVICE=eth0.1001 + IPADDR=10.0.1.2 + NETMASK=255.255.0.0 + NM_CONTROLLED=no + ONBOOT=yes + PHYSDEV=eth0 + USERCTL=no + VLAN=yes + """), + self.control_path(): dedent("""\ + NETWORKING=yes + """), + } + self._apply_and_verify( + self.distro.apply_network_config, cfg, + expected_cfgs=expected_cfgs) + class TestNetCfgDistroOpensuse(TestNetCfgDistroBase): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 54cc8469..207e47bb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1633,7 +1633,6 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PHYSDEV=bond0 - TYPE=Ethernet USERCTL=no VLAN=yes"""), 'ifcfg-br0': textwrap.dedent("""\ @@ -1677,7 +1676,6 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PHYSDEV=eth0 - TYPE=Ethernet USERCTL=no VLAN=yes"""), 'ifcfg-eth1': textwrap.dedent("""\ @@ -2286,7 +2284,6 @@ iface bond0 inet6 static NM_CONTROLLED=no ONBOOT=yes PHYSDEV=en0 - TYPE=Ethernet USERCTL=no VLAN=yes"""), }, @@ -3339,7 +3336,6 @@ USERCTL=no NM_CONTROLLED=no ONBOOT=yes PHYSDEV=eno1 - TYPE=Ethernet USERCTL=no VLAN=yes """) -- cgit v1.2.3 From 839016e3014d783354bc380799d914ff81ee4efa Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 15 Sep 2020 15:00:03 -0400 Subject: user_data: remove unused constant (#566) This was added in d00126c167fc06d913d99cfc184bf3402cb8cf53, but not removed in ef041fd822a2cf3a4022525e942ce988b1f95180 which removed the one usage of it from the original commit. --- cloudinit/user_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f234b962..1317e063 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -26,7 +26,6 @@ LOG = logging.getLogger(__name__) NOT_MULTIPART_TYPE = handlers.NOT_MULTIPART_TYPE PART_FN_TPL = handlers.PART_FN_TPL OCTET_TYPE = handlers.OCTET_TYPE -INCLUDE_MAP = handlers.INCLUSION_TYPES_MAP # Saves typing errors CONTENT_TYPE = 'Content-Type' -- cgit v1.2.3 From 6d332e5c8dbfb6521a530b1fa49d73da51efff96 Mon Sep 17 00:00:00 2001 From: Emmanuel Thomé Date: Tue, 15 Sep 2020 21:51:52 +0200 Subject: create a shutdown_command method in distro classes (#567) Under FreeBSD, we want to use "shutdown -p" for poweroff. Alpine Linux also has some specificities. We choose to define a method that returns the shutdown command line to use, rather than a method that actually does the shutdown. This makes it easier to have the tests in test_handler_power_state do their verifications. Two tests are added for the special behaviours that are known so far. --- cloudinit/config/cc_power_state_change.py | 56 +++------------------ cloudinit/distros/__init__.py | 19 +++++++ cloudinit/distros/alpine.py | 26 ++++++++++ cloudinit/distros/bsd.py | 4 ++ .../test_handler/test_handler_power_state.py | 58 ++++++++++++++++------ tools/.github-cla-signers | 1 + 6 files changed, 102 insertions(+), 62 deletions(-) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 6fcb8a7d..b0cfafcd 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -117,7 +117,7 @@ def check_condition(cond, log=None): def handle(_name, cfg, cloud, log, _args): try: - (args, timeout, condition) = load_power_state(cfg, cloud.distro.name) + (args, timeout, condition) = load_power_state(cfg, cloud.distro) if args is None: log.debug("no power_state provided. doing nothing") return @@ -144,19 +144,7 @@ def handle(_name, cfg, cloud, log, _args): condition, execmd, [args, devnull_fp]) -def convert_delay(delay, fmt=None, scale=None): - if not fmt: - fmt = "+%s" - if not scale: - scale = 1 - - if delay != "now": - delay = fmt % int(int(delay) * int(scale)) - - return delay - - -def load_power_state(cfg, distro_name): +def load_power_state(cfg, distro): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found pstate = cfg.get('power_state') @@ -167,44 +155,16 @@ def load_power_state(cfg, distro_name): if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") - opt_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} - + modes_ok = ['halt', 'poweroff', 'reboot'] mode = pstate.get("mode") - if mode not in opt_map: + if mode not in distro.shutdown_options_map: raise TypeError( "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(opt_map.keys()), mode)) - - delay = pstate.get("delay", "now") - message = pstate.get("message") - scale = 1 - fmt = "+%s" - command = ["shutdown", opt_map[mode]] - - if distro_name == 'alpine': - # Convert integer 30 or string '30' to '1800' (seconds) as Alpine's - # halt/poweroff/reboot commands take seconds rather than minutes. - scale = 60 - # No "+" in front of delay value as not supported by Alpine's commands. - fmt = "%s" - if delay == "now": - # Alpine's commands do not understand "now". - delay = "0" - command = [mode, "-d"] - # Alpine's commands don't support a message. - message = None - - try: - delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError as e: - raise TypeError( - "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay - ) from e + (','.join(modes_ok), mode)) - args = command + [delay] - if message: - args.append(message) + args = distro.shutdown_command(mode=mode, + delay=pstate.get("delay", "now"), + message=pstate.get("message")) try: timeout = float(pstate.get('timeout', 30.0)) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2bd9bae8..fac8cf67 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -73,6 +73,9 @@ class Distro(metaclass=abc.ABCMeta): renderer_configs = {} _preferred_ntp_clients = None networking_cls = LinuxNetworking + # This is used by self.shutdown_command(), and can be overridden in + # subclasses + shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} def __init__(self, name, cfg, paths): self._paths = paths @@ -750,6 +753,22 @@ class Distro(metaclass=abc.ABCMeta): subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) + def shutdown_command(self, *, mode, delay, message): + # called from cc_power_state_change.load_power_state + command = ["shutdown", self.shutdown_options_map[mode]] + try: + if delay != "now": + delay = "+%d" % int(delay) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + args = command + [delay] + if message: + args.append(message) + return args + def _apply_hostname_transformations_to_url(url: str, transformations: list): """ diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py index e42443fc..e92ff3fb 100644 --- a/cloudinit/distros/alpine.py +++ b/cloudinit/distros/alpine.py @@ -162,4 +162,30 @@ class Distro(distros.Distro): return self._preferred_ntp_clients + def shutdown_command(self, mode='poweroff', delay='now', message=None): + # called from cc_power_state_change.load_power_state + # Alpine has halt/poweroff/reboot, with the following specifics: + # - we use them rather than the generic "shutdown" + # - delay is given with "-d [integer]" + # - the integer is in seconds, cannot be "now", and takes no "+" + # - no message is supported (argument ignored, here) + + command = [mode, "-d"] + + # Convert delay from minutes to seconds, as Alpine's + # halt/poweroff/reboot commands take seconds rather than minutes. + if delay == "now": + # Alpine's commands do not understand "now". + command += ['0'] + else: + try: + command.append(str(int(delay) * 60)) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + + return command + # vi: ts=4 expandtab diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 2ed7a7d5..f717a667 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -17,6 +17,10 @@ class BSD(distros.Distro): hostname_conf_fn = '/etc/rc.conf' rc_conf_fn = "/etc/rc.conf" + # This differs from the parent Distro class, which has -P for + # poweroff. + shutdown_options_map = {'halt': '-H', 'poweroff': '-p', 'reboot': '-r'} + # Set in BSD distro subclasses group_add_cmd_prefix = [] pkg_cmd_install_prefix = [] diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 93b24fdc..4ac49424 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,72 +4,102 @@ import sys from cloudinit.config import cc_power_state_change as psc +from cloudinit import distros +from cloudinit import helpers + from cloudinit.tests import helpers as t_help from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): + def setUp(self): + super(TestLoadPowerState, self).setUp() + cls = distros.fetch('ubuntu') + paths = helpers.Paths({}) + self.dist = cls('ubuntu', {}, paths) + def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout, _condition) = psc.load_power_state({}, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state({}, self.dist) self.assertIsNone(cmd) def test_irrelevant_config(self): # no power_state field in config should return None for cmd (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}, - 'ubuntu') + self.dist) self.assertIsNone(cmd) def test_invalid_mode(self): + cfg = {'power_state': {'mode': 'gibberish'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) cfg = {'power_state': {'mode': ''}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_empty_mode(self): cfg = {'power_state': {'message': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_modes(self): cfg = {'power_state': {}} for mode in ('halt', 'poweroff', 'reboot'): cfg['power_state']['mode'] = mode - check_lps_ret(psc.load_power_state(cfg, 'ubuntu'), mode=mode) + check_lps_ret(psc.load_power_state(cfg, self.dist), mode=mode) def test_invalid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} for delay in ("now", "+1", "+30"): cfg['power_state']['delay'] = delay - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) def test_message_present(self): cfg = {'power_state': {'mode': 'poweroff', 'message': 'GOODBYE'}} - ret = psc.load_power_state(cfg, 'ubuntu') - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + ret = psc.load_power_state(cfg, self.dist) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertIn(cfg['power_state']['message'], ret[0]) def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout, _condition) = psc.load_power_state(cfg, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state(cfg, self.dist) self.assertNotIn("", cmd) - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertTrue(len(cmd) == 3) def test_condition_null_raises(self): cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_condition_default_is_true(self): cfg = {'power_state': {'mode': 'poweroff'}} - _cmd, _timeout, cond = psc.load_power_state(cfg, 'ubuntu') + _cmd, _timeout, cond = psc.load_power_state(cfg, self.dist) self.assertEqual(cond, True) + def test_freebsd_poweroff_uses_lowercase_p(self): + cls = distros.fetch('freebsd') + paths = helpers.Paths({}) + freebsd = cls('freebsd', {}, paths) + cfg = {'power_state': {'mode': 'poweroff'}} + ret = psc.load_power_state(cfg, freebsd) + self.assertIn('-p', ret[0]) + + def test_alpine_delay(self): + # alpine takes delay in seconds. + cls = distros.fetch('alpine') + paths = helpers.Paths({}) + alpine = cls('alpine', {}, paths) + cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} + for delay, value in (('now', 0), ("+1", 60), ("+30", 1800)): + cfg['power_state']['delay'] = delay + ret = psc.load_power_state(cfg, alpine) + self.assertEqual('-d', ret[0][1]) + self.assertEqual(str(value), ret[0][2]) + class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index aa946511..f01e9b66 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -6,6 +6,7 @@ candlerb dermotbradley dhensby eandersson +emmanuelthome izzyleung johnsonshi jqueuniet -- cgit v1.2.3 From 22220e200a43f9a172e1abac93907b48d60d3ee0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 16 Sep 2020 09:53:09 -0400 Subject: cloudinit: remove unused LOG variables (#574) Co-authored-by: Rick Harding --- cloudinit/distros/alpine.py | 3 --- cloudinit/distros/amazon.py | 4 ---- cloudinit/distros/centos.py | 3 --- cloudinit/distros/fedora.py | 4 ---- cloudinit/distros/opensuse.py | 3 --- cloudinit/distros/sles.py | 4 ---- cloudinit/distros/ubuntu.py | 3 --- cloudinit/handlers/shell_script.py | 3 --- cloudinit/mergers/__init__.py | 2 -- cloudinit/sources/DataSourceBigstep.py | 3 --- cloudinit/sources/DataSourceHetzner.py | 3 --- cloudinit/sources/DataSourceNone.py | 3 --- cloudinit/sources/helpers/hetzner.py | 3 --- 13 files changed, 41 deletions(-) diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py index e92ff3fb..ca5bfe80 100644 --- a/cloudinit/distros/alpine.py +++ b/cloudinit/distros/alpine.py @@ -8,7 +8,6 @@ from cloudinit import distros from cloudinit import helpers -from cloudinit import log as logging from cloudinit import subp from cloudinit import util @@ -16,8 +15,6 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit.settings import PER_INSTANCE -LOG = logging.getLogger(__name__) - NETWORK_FILE_HEADER = """\ # This file is generated from information provided by the datasource. Changes # to it will not persist across an instance reboot. To disable cloud-init's diff --git a/cloudinit/distros/amazon.py b/cloudinit/distros/amazon.py index ff9a549f..5fcec952 100644 --- a/cloudinit/distros/amazon.py +++ b/cloudinit/distros/amazon.py @@ -12,10 +12,6 @@ from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(rhel.Distro): diff --git a/cloudinit/distros/centos.py b/cloudinit/distros/centos.py index 4b803d2e..edb3165d 100644 --- a/cloudinit/distros/centos.py +++ b/cloudinit/distros/centos.py @@ -1,9 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) class Distro(rhel.Distro): diff --git a/cloudinit/distros/fedora.py b/cloudinit/distros/fedora.py index a9490d0e..0fe1fbca 100644 --- a/cloudinit/distros/fedora.py +++ b/cloudinit/distros/fedora.py @@ -10,10 +10,6 @@ from cloudinit.distros import rhel -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(rhel.Distro): pass diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index b8e557b8..7ca0ef99 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -13,15 +13,12 @@ from cloudinit import distros from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit import helpers -from cloudinit import log as logging from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util as rhutil from cloudinit.settings import PER_INSTANCE -LOG = logging.getLogger(__name__) - class Distro(distros.Distro): clock_conf_fn = '/etc/sysconfig/clock' diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 6e336cbf..f3bfb9c2 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -6,10 +6,6 @@ from cloudinit.distros import opensuse -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - class Distro(opensuse.Distro): pass diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index b4c4b0c3..2a1f93d9 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -11,13 +11,10 @@ from cloudinit.distros import debian from cloudinit.distros import PREFERRED_NTP_CLIENTS -from cloudinit import log as logging from cloudinit import util import copy -LOG = logging.getLogger(__name__) - class Distro(debian.Distro): diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 214714bc..9917f551 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -11,13 +11,10 @@ import os from cloudinit import handlers -from cloudinit import log as logging from cloudinit import util from cloudinit.settings import (PER_ALWAYS) -LOG = logging.getLogger(__name__) - class ShellScriptPartHandler(handlers.Handler): diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 668e3cd6..7fa493a6 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -7,12 +7,10 @@ import re from cloudinit import importer -from cloudinit import log as logging from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") -LOG = logging.getLogger(__name__) DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 52fff20a..63435279 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -7,13 +7,10 @@ import errno import json -from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper from cloudinit import util -LOG = logging.getLogger(__name__) - class DataSourceBigstep(sources.DataSource): diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index a86035e0..79353882 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -6,15 +6,12 @@ """Hetzner Cloud API Documentation. https://docs.hetzner.cloud/""" -from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import sources from cloudinit import util import cloudinit.sources.helpers.hetzner as hc_helper -LOG = logging.getLogger(__name__) - BASE_URL_V1 = 'http://169.254.169.254/hetzner/v1' BUILTIN_DS_CONFIG = { diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py index e6250801..b7656ac5 100644 --- a/cloudinit/sources/DataSourceNone.py +++ b/cloudinit/sources/DataSourceNone.py @@ -4,11 +4,8 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import log as logging from cloudinit import sources -LOG = logging.getLogger(__name__) - class DataSourceNone(sources.DataSource): diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 72edb023..33dc4c53 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -3,15 +3,12 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import log as logging from cloudinit import url_helper from cloudinit import util import base64 import binascii -LOG = logging.getLogger(__name__) - def read_metadata(url, timeout=2, sec_between=2, retries=30): response = url_helper.readurl(url, timeout=timeout, -- cgit v1.2.3 From a87ccadca114696bbca1e9d681efc8ce8955f454 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Wed, 16 Sep 2020 21:19:57 +0200 Subject: opennebula.rst: minor readability improvements (#573) - use `sh` as highlight language of the code block - change order so that the confusing indentation is less confusing --- doc/rtd/topics/datasources/opennebula.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/rtd/topics/datasources/opennebula.rst b/doc/rtd/topics/datasources/opennebula.rst index 8e7c2558..350a3e93 100644 --- a/doc/rtd/topics/datasources/opennebula.rst +++ b/doc/rtd/topics/datasources/opennebula.rst @@ -122,13 +122,13 @@ OpenNebula datasource only in 'net' mode. Example VM's context section ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:: +.. code-block:: sh CONTEXT=[ - PUBLIC_IP="$NIC[IP]", SSH_KEY="$USER[SSH_KEY] $USER[SSH_KEY1] - $USER[SSH_KEY2] ", + $USER[SSH_KEY2]", + PUBLIC_IP="$NIC[IP]", USER_DATA="#cloud-config # see https://help.ubuntu.com/community/CloudInit -- cgit v1.2.3 From 5fc34d81a002f6ca0706f5285ee15b919c3d8d2e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 16 Sep 2020 16:49:34 -0400 Subject: boot.rst: add First Boot Determination section (#568) LP: #1888858 --- doc/rtd/topics/boot.rst | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/doc/rtd/topics/boot.rst b/doc/rtd/topics/boot.rst index 4e79c958..a5282e35 100644 --- a/doc/rtd/topics/boot.rst +++ b/doc/rtd/topics/boot.rst @@ -157,4 +157,90 @@ finished, the ``cloud-init status`` subcommand can help block external scripts until cloud-init is done without having to write your own systemd units dependency chains. See :ref:`cli_status` for more info. +First Boot Determination +************************ + +cloud-init has to determine whether or not the current boot is the first boot +of a new instance or not, so that it applies the appropriate configuration. On +an instance's first boot, it should run all "per-instance" configuration, +whereas on a subsequent boot it should run only "per-boot" configuration. This +section describes how cloud-init performs this determination, as well as why it +is necessary. + +When it runs, cloud-init stores a cache of its internal state for use across +stages and boots. + +If this cache is present, then cloud-init has run on this system before. +[#not-present]_ There are two cases where this could occur. Most commonly, +the instance has been rebooted, and this is a second/subsequent boot. +Alternatively, the filesystem has been attached to a *new* instance, and this +is an instance's first boot. The most obvious case where this happens is when +an instance is launched from an image captured from a launched instance. + +By default, cloud-init attempts to determine which case it is running in by +checking the instance ID in the cache against the instance ID it determines at +runtime. If they do not match, then this is an instance's first boot; +otherwise, it's a subsequent boot. Internally, cloud-init refers to this +behavior as ``check``. + +This behavior is required for images captured from launched instances to +behave correctly, and so is the default which generic cloud images ship with. +However, there are cases where it can cause problems. [#problems]_ For these +cases, cloud-init has support for modifying its behavior to trust the instance +ID that is present in the system unconditionally. This means that cloud-init +will never detect a new instance when the cache is present, and it follows that +the only way to cause cloud-init to detect a new instance (and therefore its +first boot) is to manually remove cloud-init's cache. Internally, this +behavior is referred to as ``trust``. + +To configure which of these behaviors to use, cloud-init exposes the +``manual_cache_clean`` configuration option. When ``false`` (the default), +cloud-init will ``check`` and clean the cache if the instance IDs do not match +(this is the default, as discussed above). When ``true``, cloud-init will +``trust`` the existing cache (and therefore not clean it). + +Manual Cache Cleaning +===================== + +cloud-init ships a command for manually cleaning the cache: ``cloud-init +clean``. See :ref:`cli_clean`'s documentation for further details. + +Reverting ``manual_cache_clean`` Setting +======================================== + +Currently there is no support for switching an instance that is launched with +``manual_cache_clean: true`` from ``trust`` behavior to ``check`` behavior, +other than manually cleaning the cache. + +.. warning:: If you want to capture an instance that is currently in ``trust`` + mode as an image for launching other instances, you **must** manually clean + the cache. If you do not do so, then instances launched from the captured + image will all detect their first boot as a subsequent boot of the captured + instance, and will not apply any per-instance configuration. + + This is a functional issue, but also a potential security one: cloud-init is + responsible for rotating SSH host keys on first boot, and this will not + happen on these instances. + +.. [#not-present] It follows that if this cache is not present, cloud-init has + not run on this system before, so this is unambiguously this instance's + first boot. + +.. [#problems] A couple of ways in which this strict reliance on the presence + of a datasource has been observed to cause problems: + + * If a cloud's metadata service is flaky and cloud-init cannot obtain the + instance ID locally on that platform, cloud-init's instance ID + determination will sometimes fail to determine the current instance ID, + which makes it impossible to determine if this is an instance's first or + subsequent boot (`#1885527`_). + * If cloud-init is used to provision a physical appliance or device and an + attacker can present a datasource to the device with a different instance + ID, then cloud-init's default behavior will detect this as an instance's + first boot and reset the device using the attacker's configuration + (this has been observed with the NoCloud datasource in `#1879530`_). + +.. _#1885527: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1885527 +.. _#1879530: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1879530 + .. vi: textwidth=79 -- cgit v1.2.3 From 6b5c306b537aafeded249fc82a3317fba8214508 Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Fri, 18 Sep 2020 18:02:15 +0000 Subject: Add vendor-data support to seedfrom parameter for NoCloud and OVF (#570) --- cloudinit/sources/DataSourceNoCloud.py | 3 ++- cloudinit/sources/DataSourceOVF.py | 4 +++- cloudinit/util.py | 19 +++++++++++++++++-- tests/unittests/test_util.py | 24 +++++++++++++++++++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index e408d730..d4a175e8 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -157,13 +157,14 @@ class DataSourceNoCloud(sources.DataSource): # This could throw errors, but the user told us to do it # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud, vd) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud + mydata['vendor-data'] = vd found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index e53d2eb1..a5ccb8f6 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -73,6 +73,7 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" + vd = "" vmwareImcConfigFilePath = None nicspath = None @@ -304,7 +305,7 @@ class DataSourceOVF(sources.DataSource): seedfrom, self) return False - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud, vd) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) md = util.mergemanydict([md, md_seed]) @@ -316,6 +317,7 @@ class DataSourceOVF(sources.DataSource): self.seed = ",".join(found) self.metadata = md self.userdata_raw = ud + self.vendordata_raw = vd self.cfg = cfg return True diff --git a/cloudinit/util.py b/cloudinit/util.py index 64142f23..e47f1cf6 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -761,8 +761,9 @@ def del_dir(path): # 'meta-data' entries def read_optional_seed(fill, base="", ext="", timeout=5): try: - (md, ud) = read_seeded(base, ext, timeout) + (md, ud, vd) = read_seeded(base, ext, timeout) fill['user-data'] = ud + fill['vendor-data'] = vd fill['meta-data'] = md return True except url_helper.UrlError as e: @@ -840,9 +841,11 @@ def load_yaml(blob, default=None, allowed=(dict,)): def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.find("%s") >= 0: ud_url = base % ("user-data" + ext) + vd_url = base % ("vendor-data" + ext) md_url = base % ("meta-data" + ext) else: ud_url = "%s%s%s" % (base, "user-data", ext) + vd_url = "%s%s%s" % (base, "vendor-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) md_resp = url_helper.read_file_or_url(md_url, timeout=timeout, @@ -857,7 +860,19 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if ud_resp.ok(): ud = ud_resp.contents - return (md, ud) + vd = None + try: + vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, + retries=retries) + except url_helper.UrlError as e: + LOG.debug("Error in vendor-data response: %s", e) + else: + if vd_resp.ok(): + vd = vd_resp.contents + else: + LOG.debug("Error in vendor-data response") + + return (md, ud, vd) def read_conf_d(confd): diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index fc557469..cca53123 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -735,13 +735,35 @@ class TestReadSeeded(helpers.TestCase): def test_unicode_not_messed_up(self): ud = b"userdatablob" + vd = b"vendordatablob" + helpers.populate_dir( + self.tmp, {'meta-data': "key1: val1", 'user-data': ud, + 'vendor-data': vd}) + sdir = self.tmp + os.path.sep + (found_md, found_ud, found_vd) = util.read_seeded(sdir) + + self.assertEqual(found_md, {'key1': 'val1'}) + self.assertEqual(found_ud, ud) + self.assertEqual(found_vd, vd) + + +class TestReadSeededWithoutVendorData(helpers.TestCase): + def setUp(self): + super(TestReadSeededWithoutVendorData, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + + def test_unicode_not_messed_up(self): + ud = b"userdatablob" + vd = None helpers.populate_dir( self.tmp, {'meta-data': "key1: val1", 'user-data': ud}) sdir = self.tmp + os.path.sep - (found_md, found_ud) = util.read_seeded(sdir) + (found_md, found_ud, found_vd) = util.read_seeded(sdir) self.assertEqual(found_md, {'key1': 'val1'}) self.assertEqual(found_ud, ud) + self.assertEqual(found_vd, vd) class TestEncode(helpers.TestCase): -- cgit v1.2.3 From 09a0dfb1759a4f747415c8cd6b66c63cc4374e5d Mon Sep 17 00:00:00 2001 From: Wacław Schiller Date: Mon, 21 Sep 2020 18:01:35 +0200 Subject: Fix typo in disk_setup module's description (#579) --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index a7bdc703..d1200694 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -35,7 +35,7 @@ either a size or a list containing a size and the numerical value for a partition type. The size for partitions is specified in **percentage** of disk space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). The ``overwrite`` option controls whether this module tries to be safe about -writing partition talbes or not. If ``overwrite: false`` is set, the device +writing partition tables or not. If ``overwrite: false`` is set, the device will be checked for a partition table and for a file system and if either is found, the operation will be skipped. If ``overwrite: true`` is set, no checks will be performed. -- cgit v1.2.3 From d2e1b315f1fbd325a62434c7d46bc8ea41417333 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 22 Sep 2020 09:00:48 -0400 Subject: cc_users_groups: minor doc formatting fix (#577) Co-authored-by: Rick Harding --- cloudinit/config/cc_users_groups.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 426498a3..d4e923ef 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -80,10 +80,9 @@ config keys for an entry in ``users`` are as follows: .. note:: Most of these configuration options will not be honored if the user - already exists. Following options are the exceptions and they are - applicable on already-existing users: - - 'plain_text_passwd', 'hashed_passwd', 'lock_passwd', 'sudo', - 'ssh_authorized_keys', 'ssh_redirect_user'. + already exists. The following options are the exceptions; they are applied + to already-existing users: ``plain_text_passwd``, ``hashed_passwd``, + ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, ``ssh_redirect_user``. **Internal name:** ``cc_users_groups`` -- cgit v1.2.3 From 9d9f4f323fdf978b91c4df86534b533e582da923 Mon Sep 17 00:00:00 2001 From: Wacław Schiller Date: Tue, 22 Sep 2020 16:52:05 +0200 Subject: Fix typo in resolv_conf module's description (#578) --- cloudinit/config/cc_resolv_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 519e66eb..7beb11ca 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -14,7 +14,7 @@ Resolv Conf This module is intended to manage resolv.conf in environments where early configuration of resolv.conf is necessary for further bootstrapping and/or where configuration management such as puppet or chef own dns configuration. -As Debian/Ubuntu will, by default, utilize resovlconf, and similarly RedHat +As Debian/Ubuntu will, by default, utilize resolvconf, and similarly RedHat will use sysconfig, this module is likely to be of little use unless those are configured correctly. -- cgit v1.2.3 From 53465092a590fb72447ffc0f6b7b53e6609430f4 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 23 Sep 2020 14:32:32 -0400 Subject: features: refresh docs for easier out-of-context reading (#582) --- cloudinit/features.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/cloudinit/features.py b/cloudinit/features.py index c44fa29e..e1116a17 100644 --- a/cloudinit/features.py +++ b/cloudinit/features.py @@ -21,20 +21,32 @@ all valid states of a flag, not just the default state. ERROR_ON_USER_DATA_FAILURE = True """ If there is a failure in obtaining user data (i.e., #include or -decompress fails), old behavior is to log a warning and proceed. -After the 20.2 release, we instead raise an exception. -This flag can be removed after Focal is no longer supported +decompress fails) and ``ERROR_ON_USER_DATA_FAILURE`` is ``False``, +cloud-init will log a warning and proceed. If it is ``True``, +cloud-init will instead raise an exception. + +As of 20.3, ``ERROR_ON_USER_DATA_FAILURE`` is ``True``. + +(This flag can be removed after Focal is no longer supported.) """ ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False """ -When configuring apt mirrors, old behavior is to allow -the use of ec2 mirrors if the datasource availability_zone format -matches one of the possible aws ec2 regions. After the 20.2 release, we -no longer publish ec2 region mirror urls on non-AWS cloud platforms. -Besides feature_overrides.py, users can override this by providing -#cloud-config apt directives. +When configuring apt mirrors, if +``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``True`` cloud-init +will detect that a datasource's ``availability_zone`` property looks +like an EC2 availability zone and set the ``ec2_region`` variable when +generating mirror URLs; this can lead to incorrect mirrors being +configured in clouds whose AZs follow EC2's naming pattern. + +As of 20.3, ``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``False`` +so we no longer include ``ec2_region`` in mirror determination on +non-AWS cloud platforms. + +If the old behavior is desired, users can provide the appropriate +mirrors via :py:mod:`apt: ` +directives in cloud-config. """ try: -- cgit v1.2.3 From 43164902dc97cc0c51ca1b200fa09c9303a4beee Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Thu, 24 Sep 2020 09:46:19 -0700 Subject: Azure parse_network_config uses fallback cfg when generate IMDS network cfg fails (#549) Azure datasource's `parse_network_config` throws a fatal uncaught exception when an exception is raised during generation of network config from IMDS metadata. This happens when IMDS metadata is invalid/corrupted (such as when it is missing network or interface metadata). This causes the rest of provisioning to fail. This changes `parse_network_config` to be a non-fatal implementation. Additionally, when generating network config from IMDS metadata fails, fall back on generating fallback network config (`_generate_network_config_from_fallback_config`). This also changes fallback network config generation (`_generate_network_config_from_fallback_config`) to blacklist an additional driver: `mlx5_core`. --- cloudinit/sources/DataSourceAzure.py | 154 +++++++++++++++----------- tests/unittests/test_datasource/test_azure.py | 130 ++++++++++++---------- 2 files changed, 164 insertions(+), 120 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index e98fd497..773c60d7 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1387,76 +1387,104 @@ def load_azure_ds_dir(source_dir): return (md, ud, cfg, {'ovf-env.xml': contents}) -def parse_network_config(imds_metadata): +@azure_ds_telemetry_reporter +def parse_network_config(imds_metadata) -> dict: """Convert imds_metadata dictionary to network v2 configuration. - Parses network configuration from imds metadata if present or generate fallback network config excluding mlx4_core devices. @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - with events.ReportEventStack( - name="parse_network_config", - description="", - parent=azure_ds_reporter - ) as evt: - if imds_metadata != sources.UNSET and imds_metadata: - netconfig = {'version': 2, 'ethernets': {}} - LOG.debug('Azure: generating network configuration from IMDS') - network_metadata = imds_metadata['network'] - for idx, intf in enumerate(network_metadata['interface']): - # First IPv4 and/or IPv6 address will be obtained via DHCP. - # Any additional IPs of each type will be set as static - # addresses. - nicname = 'eth{idx}'.format(idx=idx) - dhcp_override = {'route-metric': (idx + 1) * 100} - dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, - 'dhcp6': False} - for addr_type in ('ipv4', 'ipv6'): - addresses = intf.get(addr_type, {}).get('ipAddress', []) - if addr_type == 'ipv4': - default_prefix = '24' - else: - default_prefix = '128' - if addresses: - dev_config['dhcp6'] = True - # non-primary interfaces should have a higher - # route-metric (cost) so default routes prefer - # primary nic due to lower route-metric value - dev_config['dhcp6-overrides'] = dhcp_override - for addr in addresses[1:]: - # Append static address config for ip > 1 - netPrefix = intf[addr_type]['subnet'][0].get( - 'prefix', default_prefix) - privateIp = addr['privateIpAddress'] - if not dev_config.get('addresses'): - dev_config['addresses'] = [] - dev_config['addresses'].append( - '{ip}/{prefix}'.format( - ip=privateIp, prefix=netPrefix)) - if dev_config: - mac = ':'.join(re.findall(r'..', intf['macAddress'])) - dev_config.update({ - 'match': {'macaddress': mac.lower()}, - 'set-name': nicname - }) - # With netvsc, we can get two interfaces that - # share the same MAC, so we need to make sure - # our match condition also contains the driver - driver = device_driver(nicname) - if driver and driver == 'hv_netvsc': - dev_config['match']['driver'] = driver - netconfig['ethernets'][nicname] = dev_config - evt.description = "network config from imds" - else: - blacklist = ['mlx4_core'] - LOG.debug('Azure: generating fallback configuration') - # generate a network config, blacklist picking mlx4_core devs - netconfig = net.generate_fallback_config( - blacklist_drivers=blacklist, config_driver=True) - evt.description = "network config from fallback" - return netconfig + if imds_metadata != sources.UNSET and imds_metadata: + try: + return _generate_network_config_from_imds_metadata(imds_metadata) + except Exception as e: + LOG.error( + 'Failed generating network config ' + 'from IMDS network metadata: %s', str(e)) + try: + return _generate_network_config_from_fallback_config() + except Exception as e: + LOG.error('Failed generating fallback network config: %s', str(e)) + return {} + + +@azure_ds_telemetry_reporter +def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: + """Convert imds_metadata dictionary to network v2 configuration. + Parses network configuration from imds metadata. + + @param: imds_metadata: Dict of content read from IMDS network service. + @return: Dictionary containing network version 2 standard configuration. + """ + netconfig = {'version': 2, 'ethernets': {}} + network_metadata = imds_metadata['network'] + for idx, intf in enumerate(network_metadata['interface']): + # First IPv4 and/or IPv6 address will be obtained via DHCP. + # Any additional IPs of each type will be set as static + # addresses. + nicname = 'eth{idx}'.format(idx=idx) + dhcp_override = {'route-metric': (idx + 1) * 100} + dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, + 'dhcp6': False} + for addr_type in ('ipv4', 'ipv6'): + addresses = intf.get(addr_type, {}).get('ipAddress', []) + if addr_type == 'ipv4': + default_prefix = '24' + else: + default_prefix = '128' + if addresses: + dev_config['dhcp6'] = True + # non-primary interfaces should have a higher + # route-metric (cost) so default routes prefer + # primary nic due to lower route-metric value + dev_config['dhcp6-overrides'] = dhcp_override + for addr in addresses[1:]: + # Append static address config for ip > 1 + netPrefix = intf[addr_type]['subnet'][0].get( + 'prefix', default_prefix) + privateIp = addr['privateIpAddress'] + if not dev_config.get('addresses'): + dev_config['addresses'] = [] + dev_config['addresses'].append( + '{ip}/{prefix}'.format( + ip=privateIp, prefix=netPrefix)) + if dev_config: + mac = ':'.join(re.findall(r'..', intf['macAddress'])) + dev_config.update({ + 'match': {'macaddress': mac.lower()}, + 'set-name': nicname + }) + # With netvsc, we can get two interfaces that + # share the same MAC, so we need to make sure + # our match condition also contains the driver + driver = device_driver(nicname) + if driver and driver == 'hv_netvsc': + dev_config['match']['driver'] = driver + netconfig['ethernets'][nicname] = dev_config + return netconfig + + +@azure_ds_telemetry_reporter +def _generate_network_config_from_fallback_config() -> dict: + """Generate fallback network config excluding mlx4_core & mlx5_core devices. + + @return: Dictionary containing network version 2 standard configuration. + """ + # Azure Dv4 and Ev4 series VMs always have mlx5 hardware. + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/ev4-esv4-series + # Earlier D and E series VMs (such as Dv2, Dv3, and Ev3 series VMs) + # can have either mlx4 or mlx5 hardware, with the older series VMs + # having a higher chance of coming with mlx4 hardware. + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/ev3-esv3-series + blacklist = ['mlx4_core', 'mlx5_core'] + # generate a network config, blacklist picking mlx4_core and mlx5_core devs + return net.generate_fallback_config( + blacklist_drivers=blacklist, config_driver=True) @azure_ds_telemetry_reporter diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2dda9925..2b22a879 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -162,8 +162,19 @@ MOCKPATH = 'cloudinit.sources.DataSourceAzure.' class TestParseNetworkConfig(CiTestCase): maxDiff = None + fallback_config = { + 'version': 1, + 'config': [{ + 'type': 'physical', 'name': 'eth0', + 'mac_address': '00:11:22:33:44:55', + 'params': {'driver': 'hv_netsvc'}, + 'subnets': [{'type': 'dhcp'}], + }] + } - def test_single_ipv4_nic_configuration(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_single_ipv4_nic_configuration(self, m_driver): """parse_network_config emits dhcp on single nic with ipv4""" expected = {'ethernets': { 'eth0': {'dhcp4': True, @@ -173,7 +184,9 @@ class TestParseNetworkConfig(CiTestCase): 'set-name': 'eth0'}}, 'version': 2} self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) - def test_increases_route_metric_for_non_primary_nics(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_increases_route_metric_for_non_primary_nics(self, m_driver): """parse_network_config increases route-metric for each nic""" expected = {'ethernets': { 'eth0': {'dhcp4': True, @@ -200,7 +213,9 @@ class TestParseNetworkConfig(CiTestCase): imds_data['network']['interface'].append(third_intf) self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv4_and_ipv6_route_metrics_match_for_nics(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv4_and_ipv6_route_metrics_match_for_nics(self, m_driver): """parse_network_config emits matching ipv4 and ipv6 route-metrics.""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24', '2001:dead:beef::2/128'], @@ -242,7 +257,9 @@ class TestParseNetworkConfig(CiTestCase): imds_data['network']['interface'].append(third_intf) self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv4_secondary_ips_will_be_static_addrs(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv4_secondary_ips_will_be_static_addrs(self, m_driver): """parse_network_config emits primary ipv4 as dhcp others are static""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24'], @@ -262,7 +279,9 @@ class TestParseNetworkConfig(CiTestCase): } self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv6_secondary_ips_will_be_static_cidrs(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv6_secondary_ips_will_be_static_cidrs(self, m_driver): """parse_network_config emits primary ipv6 as dhcp others are static""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24', '2001:dead:beef::2/10'], @@ -301,6 +320,42 @@ class TestParseNetworkConfig(CiTestCase): }}, 'version': 2} self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + @mock.patch('cloudinit.net.generate_fallback_config') + def test_parse_network_config_uses_fallback_cfg_when_no_network_metadata( + self, m_fallback_config, m_driver): + """parse_network_config generates fallback network config when the + IMDS instance metadata is corrupted/invalid, such as when + network metadata is not present. + """ + imds_metadata_missing_network_metadata = copy.deepcopy( + NETWORK_METADATA) + del imds_metadata_missing_network_metadata['network'] + m_fallback_config.return_value = self.fallback_config + self.assertEqual( + self.fallback_config, + dsaz.parse_network_config( + imds_metadata_missing_network_metadata)) + + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + @mock.patch('cloudinit.net.generate_fallback_config') + def test_parse_network_config_uses_fallback_cfg_when_no_interface_metadata( + self, m_fallback_config, m_driver): + """parse_network_config generates fallback network config when the + IMDS instance metadata is corrupted/invalid, such as when + network interface metadata is not present. + """ + imds_metadata_missing_interface_metadata = copy.deepcopy( + NETWORK_METADATA) + del imds_metadata_missing_interface_metadata['network']['interface'] + m_fallback_config.return_value = self.fallback_config + self.assertEqual( + self.fallback_config, + dsaz.parse_network_config( + imds_metadata_missing_interface_metadata)) + class TestGetMetadataFromIMDS(HttprettyTestCase): @@ -783,7 +838,9 @@ scbus-1 on xpt0 bus 0 self.assertTrue(ret) self.assertEqual(data['agent_invoked'], cfg['agent_command']) - def test_network_config_set_from_imds(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_network_config_set_from_imds(self, m_driver): """Datasource.network_config returns IMDS network data.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {} @@ -801,7 +858,10 @@ scbus-1 on xpt0 bus 0 dsrc.get_data() self.assertEqual(expected_network_config, dsrc.network_config) - def test_network_config_set_from_imds_route_metric_for_secondary_nic(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_network_config_set_from_imds_route_metric_for_secondary_nic( + self, m_driver): """Datasource.network_config adds route-metric to secondary nics.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {} @@ -1157,8 +1217,10 @@ scbus-1 on xpt0 bus 0 self.assertEqual( [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list) + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) @mock.patch('cloudinit.net.generate_fallback_config') - def test_imds_network_config(self, mock_fallback): + def test_imds_network_config(self, mock_fallback, m_driver): """Network config is generated from IMDS network data when present.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {'HostName': "myhost", 'UserName': "myuser"} @@ -1245,55 +1307,9 @@ scbus-1 on xpt0 bus 0 netconfig = dsrc.network_config self.assertEqual(netconfig, fallback_config) - mock_fallback.assert_called_with(blacklist_drivers=['mlx4_core'], - config_driver=True) - - @mock.patch('cloudinit.net.get_interface_mac') - @mock.patch('cloudinit.net.get_devicelist') - @mock.patch('cloudinit.net.device_driver') - @mock.patch('cloudinit.net.generate_fallback_config') - def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd, - mock_devlist, mock_get_mac): - """On absent network metadata, blacklist mlx from fallback config.""" - odata = {'HostName': "myhost", 'UserName': "myuser"} - data = {'ovfcontent': construct_valid_ovf_env(data=odata), - 'sys_cfg': {}} - - fallback_config = { - 'version': 1, - 'config': [{ - 'type': 'physical', 'name': 'eth0', - 'mac_address': '00:11:22:33:44:55', - 'params': {'driver': 'hv_netsvc'}, - 'subnets': [{'type': 'dhcp'}], - }] - } - blacklist_config = { - 'type': 'physical', - 'name': 'eth1', - 'mac_address': '00:11:22:33:44:55', - 'params': {'driver': 'mlx4_core'} - } - mock_fallback.return_value = fallback_config - - mock_devlist.return_value = ['eth0', 'eth1'] - mock_dd.side_effect = [ - 'hv_netsvc', # list composition, skipped - 'mlx4_core', # list composition, match - 'mlx4_core', # config get driver name - ] - mock_get_mac.return_value = '00:11:22:33:44:55' - - dsrc = self._get_ds(data) - # Represent empty response from network imds - self.m_get_metadata_from_imds.return_value = {} - ret = dsrc.get_data() - self.assertTrue(ret) - - netconfig = dsrc.network_config - expected_config = fallback_config - expected_config['config'].append(blacklist_config) - self.assertEqual(netconfig, expected_config) + mock_fallback.assert_called_with( + blacklist_drivers=['mlx4_core', 'mlx5_core'], + config_driver=True) @mock.patch(MOCKPATH + 'subp.subp') def test_get_hostname_with_no_args(self, m_subp): -- cgit v1.2.3 From 25eeaac03f1f0e0e1c460e1d0d88b23f76c17044 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 29 Sep 2020 10:28:42 -0500 Subject: Adding a PR template (#587) --- .github/PULL_REQUEST_TEMPLATE.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..82a0fadf --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +## Proposed Commit Message + + +## Additional Context + + +## Test Steps + + +## Checklist: + + - [] My code follows the process laid out in [the documentation](https://cloudinit.readthedocs.io/en/latest/topics/hacking.html) + - [] I have updated or added any unit tests accordingly + - [] I have updated or added any documentation accordingly \ No newline at end of file -- cgit v1.2.3 From 33c6d5cda8773b383bdec881c4e67f0d6c12ebd6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 1 Oct 2020 12:34:50 -0400 Subject: Fix name of ntp and chrony service on CentOS and RHEL. (#589) The service installed by the CentOS and RHEL 'ntp' package is ntpd.service not ntp.service Fix that for those two distros. Also fix chrony service from 'chrony' to 'chronyd'. LP: #1897915 --- cloudinit/config/cc_ntp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3d7279d6..e183993f 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -80,6 +80,14 @@ DISTRO_CLIENT_CONFIG = { 'confpath': '/etc/chrony/chrony.conf', }, }, + 'rhel': { + 'ntp': { + 'service_name': 'ntpd', + }, + 'chrony': { + 'service_name': 'chronyd', + }, + }, 'opensuse': { 'chrony': { 'service_name': 'chronyd', -- cgit v1.2.3 From 82ffc53273927bfc8d71e7f0c858753552d85cf1 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 1 Oct 2020 15:32:35 -0500 Subject: Initial implementation of integration testing infrastructure (#581) --- .gitignore | 3 + HACKING.rst | 11 +- doc/rtd/index.rst | 3 +- doc/rtd/topics/cloud_tests.rst | 761 ++++++++++++++++++++++++ doc/rtd/topics/integration_tests.rst | 81 +++ doc/rtd/topics/tests.rst | 758 ----------------------- integration-requirements.txt | 2 + tests/integration_tests/conftest.py | 106 ++++ tests/integration_tests/integration_settings.py | 95 +++ tests/integration_tests/platforms.py | 235 ++++++++ tox.ini | 7 + 11 files changed, 1302 insertions(+), 760 deletions(-) create mode 100644 doc/rtd/topics/cloud_tests.rst create mode 100644 doc/rtd/topics/integration_tests.rst delete mode 100644 doc/rtd/topics/tests.rst create mode 100644 tests/integration_tests/conftest.py create mode 100644 tests/integration_tests/integration_settings.py create mode 100644 tests/integration_tests/platforms.py diff --git a/.gitignore b/.gitignore index 3589b210..5a68bff9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ cloud-init_*.dsc cloud-init_*.orig.tar.gz cloud-init_*.tar.xz cloud-init_*.upload + +# user test settings +tests/integration_tests/user_settings.py diff --git a/HACKING.rst b/HACKING.rst index 60c7b5e0..4ae7f7b4 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -173,9 +173,18 @@ Cloud Config Modules * Any new modules should use underscores in any new config options and not hyphens (e.g. `new_option` and *not* `new-option`). -Unit Testing +.. _unit_testing: + +Testing ------------ +cloud-init has both unit tests and integration tests. Unit tests can +be found in-tree alongside the source code, as well as +at ``tests/unittests``. Integration tests can be found at +``tests/integration_tests``. Documentation specifically for integration +tests can be found on the :ref:`integration_tests` page, but +the guidelines specified below apply to both types of tests. + cloud-init uses `pytest`_ to run its tests, and has tests written both as ``unittest.TestCase`` sub-classes and as un-subclassed pytest tests. The following guidelines should be followed: diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 0015e35a..ddcb0b31 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -75,6 +75,7 @@ Having trouble? We would like to help! topics/dir_layout.rst topics/analyze.rst topics/docs.rst - topics/tests.rst + topics/integration_tests.rst + topics/cloud_tests.rst .. vi: textwidth=79 diff --git a/doc/rtd/topics/cloud_tests.rst b/doc/rtd/topics/cloud_tests.rst new file mode 100644 index 00000000..e4e893d2 --- /dev/null +++ b/doc/rtd/topics/cloud_tests.rst @@ -0,0 +1,761 @@ +************************ +Cloud tests (Deprecated) +************************ + +Cloud tests are longer be maintained. For writing integration +tests, see the :ref:`integration_tests` page. + +Overview +======== + +This page describes the execution, development, and architecture of the +cloud-init integration tests: + +* Execution explains the options available and running of tests +* Development shows how to write test cases +* Architecture explains the internal processes + +Execution +========= + +Overview +-------- + +In order to avoid the need for dependencies and ease the setup and +configuration users can run the integration tests via tox: + +.. code-block:: shell-session + + $ git clone https://github.com/canonical/cloud-init + $ cd cloud-init + $ tox -e citest -- -h + +Everything after the double dash will be passed to the integration tests. +Executing tests has several options: + +* ``run`` an alias to run both ``collect`` and ``verify``. The ``tree_run`` + command does the same thing, except uses a deb built from the current + working tree. + +* ``collect`` deploys on the specified platform and distro, patches with the + requested deb or rpm, and finally collects output of the arbitrary + commands. Similarly, ```tree_collect`` will collect output using a deb + built from the current working tree. + +* ``verify`` given a directory of test data, run the Python unit tests on + it to generate results. + +* ``bddeb`` will build a deb of the current working tree. + +Run +--- + +The first example will provide a complete end-to-end run of data +collection and verification. There are additional examples below +explaining how to run one or the other independently. + +.. code-block:: shell-session + + $ git clone https://github.com/canonical/cloud-init + $ cd cloud-init + $ tox -e citest -- run --verbose \ + --os-name stretch --os-name xenial \ + --deb cloud-init_0.7.8~my_patch_all.deb \ + --preserve-data --data-dir ~/collection \ + --preserve-instance + +The above command will do the following: + +* ``run`` both collect output and run tests the output + +* ``--verbose`` verbose output + +* ``--os-name stretch`` on the Debian Stretch release + +* ``--os-name xenial`` on the Ubuntu Xenial release + +* ``--deb cloud-init_0.7.8~patch_all.deb`` use this deb as the version of + cloud-init to run with + +* ``--preserve-data`` always preserve collected data, do not remove data + after successful test run + +* ``--preserve-instance`` do not destroy the instance after test to allow + for debugging the stopped instance during integration test development. By + default, test instances are destroyed after the test completes. + +* ``--data-dir ~/collection`` write collected data into `~/collection`, + rather than using a temporary directory + +For a more detailed explanation of each option see below. + +.. note:: + By default, data collected by the run command will be written into a + temporary directory and deleted after a successful. If you would + like to preserve this data, please use the option ``--preserve-data``. + +Collect +------- + +If developing tests it may be necessary to see if cloud-config works as +expected and the correct files are pulled down. In this case only a +collect can be ran by running: + +.. code-block:: shell-session + + $ tox -e citest -- collect -n xenial --data-dir /tmp/collection + +The above command will run the collection tests on xenial and place +all results into `/tmp/collection`. + +Verify +------ + +When developing tests it is much easier to simply rerun the verify scripts +without the more lengthy collect process. This can be done by running: + +.. code-block:: shell-session + + $ tox -e citest -- verify --data-dir /tmp/collection + +The above command will run the verify scripts on the data discovered in +`/tmp/collection`. + +TreeRun and TreeCollect +----------------------- + +If working on a cloud-init feature or resolving a bug, it may be useful to +run the current copy of cloud-init in the integration testing environment. +The integration testing suite can automatically build a deb based on the +current working tree of cloud-init and run the test suite using this deb. + +The ``tree_run`` and ``tree_collect`` commands take the same arguments as +the ``run`` and ``collect`` commands. These commands will build a deb and +write it into a temporary file, then start the test suite and pass that deb +in. To build a deb only, and not run the test suite, the ``bddeb`` command +can be used. + +Note that code in the cloud-init working tree that has not been committed +when the cloud-init deb is built will still be included. To build a +cloud-init deb from or use the ``tree_run`` command using a copy of +cloud-init located in a different directory, use the option ``--cloud-init +/path/to/cloud-init``. + +.. code-block:: shell-session + + $ tox -e citest -- tree_run --verbose \ + --os-name xenial --os-name stretch \ + --test modules/final_message --test modules/write_files \ + --result /tmp/result.yaml + +Bddeb +----- + +The ``bddeb`` command can be used to generate a deb file. This is used by +the tree_run and tree_collect commands to build a deb of the current +working tree. It can also be used a user to generate a deb for use in other +situations and avoid needing to have all the build and test dependencies +installed locally. + +* ``--bddeb-args``: arguments to pass through to bddeb +* ``--build-os``: distribution to use as build system (default is xenial) +* ``--build-platform``: platform to use for build system (default is lxd) +* ``--cloud-init``: path to base of cloud-init tree (default is '.') +* ``--deb``: path to write output deb to (default is '.') + +Setup Image +----------- + +By default an image that is used will remain unmodified, but certain +scenarios may require image modification. For example, many images may use +a much older cloud-init. As a result tests looking at newer functionality +will fail because a newer version of cloud-init may be required. The +following options can be used for further customization: + +* ``--deb``: install the specified deb into the image +* ``--rpm``: install the specified rpm into the image +* ``--repo``: enable a repository and upgrade cloud-init afterwards +* ``--ppa``: enable a ppa and upgrade cloud-init afterwards +* ``--upgrade``: upgrade cloud-init from repos +* ``--upgrade-full``: run a full system upgrade +* ``--script``: execute a script in the image. This can perform any setup + required that is not covered by the other options + +Test Case Development +===================== + +Overview +-------- + +As a test writer you need to develop a test configuration and a +verification file: + + * The test configuration specifies a specific cloud-config to be used by + cloud-init and a list of arbitrary commands to capture the output of + (e.g my_test.yaml) + + * The verification file runs tests on the collected output to determine + the result of the test (e.g. my_test.py) + +The names must match, however the extensions will of course be different, +yaml vs py. + +Configuration +------------- + +The test configuration is a YAML file such as *ntp_server.yaml* below: + +.. code-block:: yaml + + # + # Empty NTP config to setup using defaults + # + # NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' + # NOTE: this should not require no_ntpdate feature, use 'which' to check for + # installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' + # NOTE: the verifier should check for any ntp server not 'ubuntu.pool.ntp.org' + cloud_config: | + #cloud-config + ntp: + servers: + - pool.ntp.org + required_features: + - apt + - no_ntpdate + - ubuntu_ntp + collect_scripts: + ntp_installed_servers: | + #!/bin/bash + dpkg -l | grep ntp | wc -l + ntp_conf_dist_servers: | + #!/bin/bash + ls /etc/ntp.conf.dist | wc -l + ntp_conf_servers: | + #!/bin/bash + cat /etc/ntp.conf | grep '^server' + +There are several keys, 1 required and some optional, in the YAML file: + +1. The required key is ``cloud_config``. This should be a string of valid + YAML that is exactly what would normally be placed in a cloud-config + file, including the cloud-config header. This essentially sets up the + scenario under test. + +2. One optional key is ``collect_scripts``. This key has one or more + sub-keys containing strings of arbitrary commands to execute (e.g. + ```cat /var/log/cloud-config-output.log```). In the example above the + output of dpkg is captured, grep for ntp, and the number of lines + reported. The name of the sub-key is important. The sub-key is used by + the verification script to recall the output of the commands ran. + +3. The optional ``enabled`` key enables or disables the test case. By + default the test case will be enabled. + +4. The optional ``required_features`` key may be used to specify a list + of features flags that an image must have to be able to run the test + case. For example, if a test case relies on an image supporting apt, + then the config for the test case should include ``required_features: + [ apt ]``. + + +Default Collect Scripts +----------------------- + +By default the following files will be collected for every test. There is +no need to specify these items: + +* ``/var/log/cloud-init.log`` +* ``/var/log/cloud-init-output.log`` +* ``/run/cloud-init/.instance-id`` +* ``/run/cloud-init/result.json`` +* ``/run/cloud-init/status.json`` +* ```dpkg-query -W -f='${Version}' cloud-init``` + +Verification +------------ + +The verification script is a Python file with unit tests like the one, +`ntp_server.py`, below: + +.. code-block:: python + + # This file is part of cloud-init. See LICENSE file for license information. + + """cloud-init Integration Test Verify Script""" + from tests.cloud_tests.testcases import base + + + class TestNtp(base.CloudTestCase): + """Test ntp module""" + + def test_ntp_installed(self): + """Test ntp installed""" + out = self.get_data_file('ntp_installed_empty') + self.assertEqual(1, int(out)) + + def test_ntp_dist_entries(self): + """Test dist config file has one entry""" + out = self.get_data_file('ntp_conf_dist_empty') + self.assertEqual(1, int(out)) + + def test_ntp_entires(self): + """Test config entries""" + out = self.get_data_file('ntp_conf_empty') + self.assertIn('pool 0.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 1.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 2.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 3.ubuntu.pool.ntp.org iburst', out) + + # vi: ts=4 expandtab + + +Here is a breakdown of the unit test file: + +* The import statement allows access to the output files. + +* The class can be named anything, but must import the + ``base.CloudTestCase``, either directly or via another test class. + +* There can be 1 to N number of functions with any name, however only + functions starting with ``test_*`` will be executed. + +* There can be 1 to N number of classes in a test module, however only + classes inheriting from ``base.CloudTestCase`` will be loaded. + +* Output from the commands can be accessed via + ``self.get_data_file('key')`` where key is the sub-key of + ``collect_scripts`` above. + +* The cloud config that the test ran with can be accessed via + ``self.cloud_config``, or any entry from the cloud config can be accessed + via ``self.get_config_entry('key')``. + +* See the base ``CloudTestCase`` for additional helper functions. + +Layout +------ + +Integration tests are located under the `tests/cloud_tests` directory. +Test configurations are placed under `configs` and the test verification +scripts under `testcases`: + +.. code-block:: shell-session + + cloud-init$ tree -d tests/cloud_tests/ + tests/cloud_tests/ + ├── configs + │   ├── bugs + │   ├── examples + │   ├── main + │   └── modules + └── testcases + ├── bugs + ├── examples + ├── main + └── modules + +The sub-folders of bugs, examples, main, and modules help organize the +tests. View the README.md in each to understand in more detail each +directory. + +Test Creation Helper +-------------------- + +The integration testing suite has a built in helper to aid in test +development. Help can be invoked via ``tox -e citest -- create --help``. It +can create a template test case config file with user data passed in from +the command line, as well as a template test case verifier module. + +The following would create a test case named ``example`` under the +``modules`` category with the given description, and cloud config data read +in from ``/tmp/user_data``. + +.. code-block:: shell-session + + $ tox -e citest -- create modules/example \ + -d "a simple example test case" -c "$(< /tmp/user_data)" + + +Development Checklist +--------------------- + +* Configuration File + * Named 'your_test.yaml' + * Contains at least a valid cloud-config + * Optionally, commands to capture additional output + * Valid YAML + * Placed in the appropriate sub-folder in the configs directory + * Any image features required for the test are specified +* Verification File + * Named 'your_test.py' + * Valid unit tests validating output collected + * Passes pylint & pep8 checks + * Placed in the appropriate sub-folder in the test cases directory +* Tested by running the test: + + .. code-block:: shell-session + + $ tox -e citest -- run -verbose \ + --os-name \ + --test modules/your_test.yaml \ + [--deb ] + + +Platforms +========= + +EC2 +--- +To run on the EC2 platform it is required that the user has an AWS credentials +configuration file specifying his or her access keys and a default region. +These configuration files are the standard that the AWS cli and other AWS +tools utilize for interacting directly with AWS itself and are normally +generated when running ``aws configure``: + +.. code-block:: shell-session + + $ cat $HOME/.aws/credentials + [default] + aws_access_key_id = + aws_secret_access_key = + +.. code-block:: shell-session + + $ cat $HOME/.aws/config + [default] + region = us-west-2 + + +Azure Cloud +----------- + +To run on Azure Cloud platform users login with Service Principal and export +credentials file. Region is defaulted and can be set in +``tests/cloud_tests/platforms.yaml``. The Service Principal credentials are +the standard authentication for Azure SDK to interact with Azure Services: + +Create Service Principal account or login + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --name "APP_ID" --password "STRONG-SECRET-PASSWORD" + +.. code-block:: shell-session + + $ az login --service-principal --username "APP_ID" --password "STRONG-SECRET-PASSWORD" + +Export credentials + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --sdk-auth > $HOME/.azure/credentials.json + +.. code-block:: json + + { + "clientId": "", + "clientSecret": "", + "subscriptionId": "", + "tenantId": "", + "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", + "resourceManagerEndpointUrl": "https://management.azure.com/", + "activeDirectoryGraphResourceId": "https://graph.windows.net/", + "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", + "galleryEndpointUrl": "https://gallery.azure.com/", + "managementEndpointUrl": "https://management.core.windows.net/" + } + +Set region in platforms.yaml + +.. code-block:: yaml + + azurecloud: + enabled: true + region: West US 2 + vm_size: Standard_DS1_v2 + storage_sku: standard_lrs + tag: ci + + +Architecture +============ + +The following section outlines the high-level architecture of the +integration process. + +Overview +-------- +The process flow during a complete end-to-end LXD-backed test. + +1. Configuration + * The back end and specific distro releases are verified as supported + * The test or tests that need to be run are determined either by + directory or by individual yaml + +2. Image Creation + * Acquire the request LXD image + * Install the specified cloud-init package + * Clean the image so that it does not appear to have been booted + * A snapshot of the image is created and reused by all tests + +3. Configuration + * For each test, the cloud-config is injected into a copy of the + snapshot and booted + * The framework waits for ``/var/lib/cloud/instance/boot-finished`` + (up to 120 seconds) + * All default commands are ran and output collected + * Any commands the user specified are executed and output collected + +4. Verification + * The default commands are checked for any failures, errors, and + warnings to validate basic functionality of cloud-init completed + successfully + * The user generated unit tests are then ran validating against the + collected output + +5. Results + * If any failures were detected the test suite returns a failure + * Results can be dumped in yaml format to a specified file using the + ``-r .yaml`` option + +Configuring the Test Suite +-------------------------- + +Most of the behavior of the test suite is configurable through several yaml +files. These control the behavior of the test suite's platforms, images, and +tests. The main config files for platforms, images and test cases are +``platforms.yaml``, ``releases.yaml`` and ``testcases.yaml``. + +Config handling +^^^^^^^^^^^^^^^ + +All configurable parts of the test suite use a defaults + overrides system +for managing config entries. All base config items are dictionaries. + +Merging is done on a key-by-key basis, with all keys in the default and +override represented in the final result. If a key exists both in +the defaults and the overrides, then the behavior depends on the type of data +the key refers to. If it is atomic data or a list, then the overrides will +replace the default. If the data is a dictionary then the value will be the +result of merging that dictionary from the default config and that +dictionary from the overrides. + +Merging is done using the function +``tests.cloud_tests.config.merge_config``, which can be examined for more +detail on config merging behavior. + +The following demonstrates merge behavior: + +.. code-block:: yaml + + defaults: + list_item: + - list_entry_1 + - list_entry_2 + int_item_1: 123 + int_item_2: 234 + dict_item: + subkey_1: 1 + subkey_2: 2 + subkey_dict: + subsubkey_1: a + subsubkey_2: b + + overrides: + list_item: + - overridden_list_entry + int_item_1: 0 + dict_item: + subkey_2: false + subkey_dict: + subsubkey_2: 'new value' + + result: + list_item: + - overridden_list_entry + int_item_1: 0 + int_item_2: 234 + dict_item: + subkey_1: 1 + subkey_2: false + subkey_dict: + subsubkey_1: a + subsubkey_2: 'new value' + + +Image Config +------------ + +Image configuration is handled in ``releases.yaml``. The image configuration +controls how platforms locate and acquire images, how the platforms should +interact with the images, how platforms should detect when an image has +fully booted, any options that are required to set the image up, and +features that the image supports. + +Since settings for locating an image and interacting with it differ from +platform to platform, there are 4 levels of settings available for images on +top of the default image settings. The structure of the image config file +is: + +.. code-block:: yaml + + default_release_config: + default: + ... + : + ... + : + ... + + releases: + : + : + ... + : + ... + : + ... + + +The base config is created from the overall defaults and the overrides for +the platform. The overrides are created from the default config for the +image and the platform specific overrides for the image. + +System Boot +^^^^^^^^^^^ + +The test suite must be able to test if a system has fully booted and if +cloud-init has finished running, so that running collect scripts does not +race against the target image booting. This is done using the +``system_ready_script`` and ``cloud_init_ready_script`` image config keys. + +Each of these keys accepts a small bash test statement as a string that must +return 0 or 1. Since this test statement will be added into a larger bash +statement it must be a single statement using the ``[`` test syntax. + +The default image config provides a system ready script that works for any +systemd based image. If the image is not systemd based, then a different +test statement must be provided. The default config also provides a test +for whether or not cloud-init has finished which checks for the file +``/run/cloud-init/result.json``. This should be sufficient for most systems +as writing this file is one of the last things cloud-init does. + +The setting ``boot_timeout`` controls how long, in seconds, the platform +should wait for an image to boot. If the system ready script has not +indicated that the system is fully booted within this time an error will be +raised. + +Feature Flags +^^^^^^^^^^^^^ + +Not all test cases can work on all images due to features the test case +requires not being present on that image. If a test case requires features +in an image that are not likely to be present across all distros and +platforms that the test suite supports, then the test can be skipped +everywhere it is not supported. + +Feature flags, which are names for features supported on some images, but +not all that may be required by test cases. Configuration for feature flags +is provided in ``releases.yaml`` under the ``features`` top level key. The +features config includes a list of all currently defined feature flags, +their meanings, and a list of feature groups. + +Feature groups are groups of features that many images have in common. For +example, the ``Ubuntu_specific`` feature group includes features that +should be present across most Ubuntu releases, but may or may not be for +other distros. Feature groups are specified for an image as a list under +the key ``feature_groups``. + +An image's feature flags are derived from the features groups that that +image has and any feature overrides provided. Feature overrides can be +specified under the ``features`` key which accepts a dictionary of +``{: true/false}`` mappings. If a feature is omitted from an +image's feature flags or set to false in the overrides then the test suite +will skip any tests that require that feature when using that image. + +Feature flags may be overridden at run time using the ``--feature-override`` +command line argument. It accepts a feature flag and value to set in the +format ``=true/false``. Multiple ``--feature-override`` +flags can be used, and will all be applied to all feature flags for images +used during a test. + +Setup Overrides +^^^^^^^^^^^^^^^ + +If an image requires some of the options for image setup to be used, then it +may specify overrides for the command line arguments passed into setup +image. These may be specified as a dictionary under the ``setup_overrides`` +key. When an image is set up, the arguments that control how it is set up +will be the arguments from the command line, with any entries in +``setup_overrides`` used to override these arguments. + +For example, images that do not come with cloud-init already installed +should have ``setup_overrides: {upgrade: true}`` specified so that in the +event that no additional setup options are given, cloud-init will be +installed from the image's repos before running tests. Note that if other +options such as ``--deb`` are passed in on the command line, these will +still work as expected, since apt's policy for cloud-init would prefer the +locally installed deb over an older version from the repos. + +Platform Specific Options +^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are many platform specific options in image configuration that allow +platforms to locate images and that control additional setup that the +platform may have to do to make the image usable. For information on how +these work, please consult the documentation for that platform in the +integration testing suite and the ``releases.yaml`` file for examples. + +Error Handling +-------------- + +The test suite makes an attempt to run as many tests as possible even in the +event of some failing so that automated runs collect as much data as +possible. In the event that something goes wrong while setting up for or +running a test, the test suite will attempt to continue running any tests +which have not been affected by the error. + +For example, if the test suite was told to run tests on one platform for two +releases and an error occurred setting up the first image, all tests for +that image would be skipped, and the test suite would continue to set up +the second image and run tests on it. Or, if the system does not start +properly for one test case out of many to run on that image, that test case +will be skipped and the next one will be run. + +Note that if any errors occur, the test suite will record the failure and +where it occurred in the result data and write it out to the specified +result file. + +Results +------- + +The test suite generates result data that includes how long each stage of +the test suite took and which parts were and were not successful. This data +is dumped to the log after the collect and verify stages, and may also be +written out in yaml format to a file. If part of the setup failed, the +traceback for the failure and the error message will be included in the +result file. If a test verifier finds a problem with the collected data +from a test run, the class, test function and test will be recorded in the +result data. + +Exit Codes +^^^^^^^^^^ + +The test suite counts how many errors occur throughout a run. The exit code +after a run is the number of errors that occurred. If the exit code is +non-zero then something is wrong either with the test suite, the +configuration for an image, a test case, or cloud-init itself. + +Note that the exit code does not always directly correspond to the number +of failed test cases, since in some cases, a single error during image setup +can mean that several test cases are not run. If run is used, then the exit +code will be the sum of the number of errors in the collect and verify +stages. + +Data Dir +^^^^^^^^ + +When using run, the collected data is written into a temporary directory. In +the event that all tests pass, this directory is deleted, but if a test +fails or an error occurs, this data will be left in place, and a message +will be written to the log giving the location of the data. diff --git a/doc/rtd/topics/integration_tests.rst b/doc/rtd/topics/integration_tests.rst new file mode 100644 index 00000000..aeda326c --- /dev/null +++ b/doc/rtd/topics/integration_tests.rst @@ -0,0 +1,81 @@ +.. _integration_tests: + +******************* +Integration Testing +******************* + +Overview +========= + +Integration tests are written using pytest and are located at +``tests/integration_tests``. General design principles +laid out in :ref:`unit_testing` should be followed for integration tests. + +Setup is accomplished via a set of fixtures located in +``tests/integration_tests/conftest.py``. + +Image Setup +=========== + +Image setup occurs once when a test session begins and is implemented +via fixture. Image setup roughly follows these steps: + +* Launch an instance on the specified test platform +* Install the version of cloud-init under test +* Run ``cloud-init clean`` on the instance so subsequent boots + resemble out of the box behavior +* Take a snapshot of the instance to be used as a new image from + which new instances can be launched + +Test Setup +============== +Test setup occurs between image setup and test execution. Test setup +is implemented via one of the ``client`` fixtures. When a client fixture +is used, a test instance from which to run tests is launched prior to +test execution and torn down after. + +Test Definition +=============== +Tests are defined like any other pytest test. The ``user_data`` +mark can be used to supply the cloud-config user data. Platform specific +marks can be used to limit tests to particular platforms. The +client fixture can be used to interact with the launched +test instance. + +A basic example: + +.. code-block:: python + + USER_DATA = """#cloud-config + bootcmd: + - echo 'hello config!' > /tmp/user_data.txt""" + + + class TestSimple: + @pytest.mark.user_data(USER_DATA) + @pytest.mark.ec2 + def test_simple(self, client): + print(client.exec('cloud-init -v')) + +Test Execution +============== +Test execution happens via pytest. To run all integration tests, +you would run: + +.. code-block:: bash + + pytest tests/integration_tests/ + + +Configuration +============= + +All possible configuration values are defined in +``tests/integration_tests/integration_settings.py``. Defaults can be +overridden by supplying values in ``tests/integration_tests/user_settings.py`` +or by providing an environment variable of the same name prepended with +``CLOUD_INIT_``. For example, to set the ``PLATFORM`` setting: + +.. code-block:: bash + + CLOUD_INIT_PLATFORM='ec2' pytest tests/integration_tests/ diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst deleted file mode 100644 index f03b5969..00000000 --- a/doc/rtd/topics/tests.rst +++ /dev/null @@ -1,758 +0,0 @@ -******************* -Integration Testing -******************* - -Overview -======== - -This page describes the execution, development, and architecture of the -cloud-init integration tests: - -* Execution explains the options available and running of tests -* Development shows how to write test cases -* Architecture explains the internal processes - -Execution -========= - -Overview --------- - -In order to avoid the need for dependencies and ease the setup and -configuration users can run the integration tests via tox: - -.. code-block:: shell-session - - $ git clone https://github.com/canonical/cloud-init - $ cd cloud-init - $ tox -e citest -- -h - -Everything after the double dash will be passed to the integration tests. -Executing tests has several options: - -* ``run`` an alias to run both ``collect`` and ``verify``. The ``tree_run`` - command does the same thing, except uses a deb built from the current - working tree. - -* ``collect`` deploys on the specified platform and distro, patches with the - requested deb or rpm, and finally collects output of the arbitrary - commands. Similarly, ```tree_collect`` will collect output using a deb - built from the current working tree. - -* ``verify`` given a directory of test data, run the Python unit tests on - it to generate results. - -* ``bddeb`` will build a deb of the current working tree. - -Run ---- - -The first example will provide a complete end-to-end run of data -collection and verification. There are additional examples below -explaining how to run one or the other independently. - -.. code-block:: shell-session - - $ git clone https://github.com/canonical/cloud-init - $ cd cloud-init - $ tox -e citest -- run --verbose \ - --os-name stretch --os-name xenial \ - --deb cloud-init_0.7.8~my_patch_all.deb \ - --preserve-data --data-dir ~/collection \ - --preserve-instance - -The above command will do the following: - -* ``run`` both collect output and run tests the output - -* ``--verbose`` verbose output - -* ``--os-name stretch`` on the Debian Stretch release - -* ``--os-name xenial`` on the Ubuntu Xenial release - -* ``--deb cloud-init_0.7.8~patch_all.deb`` use this deb as the version of - cloud-init to run with - -* ``--preserve-data`` always preserve collected data, do not remove data - after successful test run - -* ``--preserve-instance`` do not destroy the instance after test to allow - for debugging the stopped instance during integration test development. By - default, test instances are destroyed after the test completes. - -* ``--data-dir ~/collection`` write collected data into `~/collection`, - rather than using a temporary directory - -For a more detailed explanation of each option see below. - -.. note:: - By default, data collected by the run command will be written into a - temporary directory and deleted after a successful. If you would - like to preserve this data, please use the option ``--preserve-data``. - -Collect -------- - -If developing tests it may be necessary to see if cloud-config works as -expected and the correct files are pulled down. In this case only a -collect can be ran by running: - -.. code-block:: shell-session - - $ tox -e citest -- collect -n xenial --data-dir /tmp/collection - -The above command will run the collection tests on xenial and place -all results into `/tmp/collection`. - -Verify ------- - -When developing tests it is much easier to simply rerun the verify scripts -without the more lengthy collect process. This can be done by running: - -.. code-block:: shell-session - - $ tox -e citest -- verify --data-dir /tmp/collection - -The above command will run the verify scripts on the data discovered in -`/tmp/collection`. - -TreeRun and TreeCollect ------------------------ - -If working on a cloud-init feature or resolving a bug, it may be useful to -run the current copy of cloud-init in the integration testing environment. -The integration testing suite can automatically build a deb based on the -current working tree of cloud-init and run the test suite using this deb. - -The ``tree_run`` and ``tree_collect`` commands take the same arguments as -the ``run`` and ``collect`` commands. These commands will build a deb and -write it into a temporary file, then start the test suite and pass that deb -in. To build a deb only, and not run the test suite, the ``bddeb`` command -can be used. - -Note that code in the cloud-init working tree that has not been committed -when the cloud-init deb is built will still be included. To build a -cloud-init deb from or use the ``tree_run`` command using a copy of -cloud-init located in a different directory, use the option ``--cloud-init -/path/to/cloud-init``. - -.. code-block:: shell-session - - $ tox -e citest -- tree_run --verbose \ - --os-name xenial --os-name stretch \ - --test modules/final_message --test modules/write_files \ - --result /tmp/result.yaml - -Bddeb ------ - -The ``bddeb`` command can be used to generate a deb file. This is used by -the tree_run and tree_collect commands to build a deb of the current -working tree. It can also be used a user to generate a deb for use in other -situations and avoid needing to have all the build and test dependencies -installed locally. - -* ``--bddeb-args``: arguments to pass through to bddeb -* ``--build-os``: distribution to use as build system (default is xenial) -* ``--build-platform``: platform to use for build system (default is lxd) -* ``--cloud-init``: path to base of cloud-init tree (default is '.') -* ``--deb``: path to write output deb to (default is '.') - -Setup Image ------------ - -By default an image that is used will remain unmodified, but certain -scenarios may require image modification. For example, many images may use -a much older cloud-init. As a result tests looking at newer functionality -will fail because a newer version of cloud-init may be required. The -following options can be used for further customization: - -* ``--deb``: install the specified deb into the image -* ``--rpm``: install the specified rpm into the image -* ``--repo``: enable a repository and upgrade cloud-init afterwards -* ``--ppa``: enable a ppa and upgrade cloud-init afterwards -* ``--upgrade``: upgrade cloud-init from repos -* ``--upgrade-full``: run a full system upgrade -* ``--script``: execute a script in the image. This can perform any setup - required that is not covered by the other options - -Test Case Development -===================== - -Overview --------- - -As a test writer you need to develop a test configuration and a -verification file: - - * The test configuration specifies a specific cloud-config to be used by - cloud-init and a list of arbitrary commands to capture the output of - (e.g my_test.yaml) - - * The verification file runs tests on the collected output to determine - the result of the test (e.g. my_test.py) - -The names must match, however the extensions will of course be different, -yaml vs py. - -Configuration -------------- - -The test configuration is a YAML file such as *ntp_server.yaml* below: - -.. code-block:: yaml - - # - # Empty NTP config to setup using defaults - # - # NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' - # NOTE: this should not require no_ntpdate feature, use 'which' to check for - # installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' - # NOTE: the verifier should check for any ntp server not 'ubuntu.pool.ntp.org' - cloud_config: | - #cloud-config - ntp: - servers: - - pool.ntp.org - required_features: - - apt - - no_ntpdate - - ubuntu_ntp - collect_scripts: - ntp_installed_servers: | - #!/bin/bash - dpkg -l | grep ntp | wc -l - ntp_conf_dist_servers: | - #!/bin/bash - ls /etc/ntp.conf.dist | wc -l - ntp_conf_servers: | - #!/bin/bash - cat /etc/ntp.conf | grep '^server' - -There are several keys, 1 required and some optional, in the YAML file: - -1. The required key is ``cloud_config``. This should be a string of valid - YAML that is exactly what would normally be placed in a cloud-config - file, including the cloud-config header. This essentially sets up the - scenario under test. - -2. One optional key is ``collect_scripts``. This key has one or more - sub-keys containing strings of arbitrary commands to execute (e.g. - ```cat /var/log/cloud-config-output.log```). In the example above the - output of dpkg is captured, grep for ntp, and the number of lines - reported. The name of the sub-key is important. The sub-key is used by - the verification script to recall the output of the commands ran. - -3. The optional ``enabled`` key enables or disables the test case. By - default the test case will be enabled. - -4. The optional ``required_features`` key may be used to specify a list - of features flags that an image must have to be able to run the test - case. For example, if a test case relies on an image supporting apt, - then the config for the test case should include ``required_features: - [ apt ]``. - - -Default Collect Scripts ------------------------ - -By default the following files will be collected for every test. There is -no need to specify these items: - -* ``/var/log/cloud-init.log`` -* ``/var/log/cloud-init-output.log`` -* ``/run/cloud-init/.instance-id`` -* ``/run/cloud-init/result.json`` -* ``/run/cloud-init/status.json`` -* ```dpkg-query -W -f='${Version}' cloud-init``` - -Verification ------------- - -The verification script is a Python file with unit tests like the one, -`ntp_server.py`, below: - -.. code-block:: python - - # This file is part of cloud-init. See LICENSE file for license information. - - """cloud-init Integration Test Verify Script""" - from tests.cloud_tests.testcases import base - - - class TestNtp(base.CloudTestCase): - """Test ntp module""" - - def test_ntp_installed(self): - """Test ntp installed""" - out = self.get_data_file('ntp_installed_empty') - self.assertEqual(1, int(out)) - - def test_ntp_dist_entries(self): - """Test dist config file has one entry""" - out = self.get_data_file('ntp_conf_dist_empty') - self.assertEqual(1, int(out)) - - def test_ntp_entires(self): - """Test config entries""" - out = self.get_data_file('ntp_conf_empty') - self.assertIn('pool 0.ubuntu.pool.ntp.org iburst', out) - self.assertIn('pool 1.ubuntu.pool.ntp.org iburst', out) - self.assertIn('pool 2.ubuntu.pool.ntp.org iburst', out) - self.assertIn('pool 3.ubuntu.pool.ntp.org iburst', out) - - # vi: ts=4 expandtab - - -Here is a breakdown of the unit test file: - -* The import statement allows access to the output files. - -* The class can be named anything, but must import the - ``base.CloudTestCase``, either directly or via another test class. - -* There can be 1 to N number of functions with any name, however only - functions starting with ``test_*`` will be executed. - -* There can be 1 to N number of classes in a test module, however only - classes inheriting from ``base.CloudTestCase`` will be loaded. - -* Output from the commands can be accessed via - ``self.get_data_file('key')`` where key is the sub-key of - ``collect_scripts`` above. - -* The cloud config that the test ran with can be accessed via - ``self.cloud_config``, or any entry from the cloud config can be accessed - via ``self.get_config_entry('key')``. - -* See the base ``CloudTestCase`` for additional helper functions. - -Layout ------- - -Integration tests are located under the `tests/cloud_tests` directory. -Test configurations are placed under `configs` and the test verification -scripts under `testcases`: - -.. code-block:: shell-session - - cloud-init$ tree -d tests/cloud_tests/ - tests/cloud_tests/ - ├── configs - │   ├── bugs - │   ├── examples - │   ├── main - │   └── modules - └── testcases - ├── bugs - ├── examples - ├── main - └── modules - -The sub-folders of bugs, examples, main, and modules help organize the -tests. View the README.md in each to understand in more detail each -directory. - -Test Creation Helper --------------------- - -The integration testing suite has a built in helper to aid in test -development. Help can be invoked via ``tox -e citest -- create --help``. It -can create a template test case config file with user data passed in from -the command line, as well as a template test case verifier module. - -The following would create a test case named ``example`` under the -``modules`` category with the given description, and cloud config data read -in from ``/tmp/user_data``. - -.. code-block:: shell-session - - $ tox -e citest -- create modules/example \ - -d "a simple example test case" -c "$(< /tmp/user_data)" - - -Development Checklist ---------------------- - -* Configuration File - * Named 'your_test.yaml' - * Contains at least a valid cloud-config - * Optionally, commands to capture additional output - * Valid YAML - * Placed in the appropriate sub-folder in the configs directory - * Any image features required for the test are specified -* Verification File - * Named 'your_test.py' - * Valid unit tests validating output collected - * Passes pylint & pep8 checks - * Placed in the appropriate sub-folder in the test cases directory -* Tested by running the test: - - .. code-block:: shell-session - - $ tox -e citest -- run -verbose \ - --os-name \ - --test modules/your_test.yaml \ - [--deb ] - - -Platforms -========= - -EC2 ---- -To run on the EC2 platform it is required that the user has an AWS credentials -configuration file specifying his or her access keys and a default region. -These configuration files are the standard that the AWS cli and other AWS -tools utilize for interacting directly with AWS itself and are normally -generated when running ``aws configure``: - -.. code-block:: shell-session - - $ cat $HOME/.aws/credentials - [default] - aws_access_key_id = - aws_secret_access_key = - -.. code-block:: shell-session - - $ cat $HOME/.aws/config - [default] - region = us-west-2 - - -Azure Cloud ------------ - -To run on Azure Cloud platform users login with Service Principal and export -credentials file. Region is defaulted and can be set in -``tests/cloud_tests/platforms.yaml``. The Service Principal credentials are -the standard authentication for Azure SDK to interact with Azure Services: - -Create Service Principal account or login - -.. code-block:: shell-session - - $ az ad sp create-for-rbac --name "APP_ID" --password "STRONG-SECRET-PASSWORD" - -.. code-block:: shell-session - - $ az login --service-principal --username "APP_ID" --password "STRONG-SECRET-PASSWORD" - -Export credentials - -.. code-block:: shell-session - - $ az ad sp create-for-rbac --sdk-auth > $HOME/.azure/credentials.json - -.. code-block:: json - - { - "clientId": "", - "clientSecret": "", - "subscriptionId": "", - "tenantId": "", - "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", - "resourceManagerEndpointUrl": "https://management.azure.com/", - "activeDirectoryGraphResourceId": "https://graph.windows.net/", - "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", - "galleryEndpointUrl": "https://gallery.azure.com/", - "managementEndpointUrl": "https://management.core.windows.net/" - } - -Set region in platforms.yaml - -.. code-block:: yaml - - azurecloud: - enabled: true - region: West US 2 - vm_size: Standard_DS1_v2 - storage_sku: standard_lrs - tag: ci - - -Architecture -============ - -The following section outlines the high-level architecture of the -integration process. - -Overview --------- -The process flow during a complete end-to-end LXD-backed test. - -1. Configuration - * The back end and specific distro releases are verified as supported - * The test or tests that need to be run are determined either by - directory or by individual yaml - -2. Image Creation - * Acquire the request LXD image - * Install the specified cloud-init package - * Clean the image so that it does not appear to have been booted - * A snapshot of the image is created and reused by all tests - -3. Configuration - * For each test, the cloud-config is injected into a copy of the - snapshot and booted - * The framework waits for ``/var/lib/cloud/instance/boot-finished`` - (up to 120 seconds) - * All default commands are ran and output collected - * Any commands the user specified are executed and output collected - -4. Verification - * The default commands are checked for any failures, errors, and - warnings to validate basic functionality of cloud-init completed - successfully - * The user generated unit tests are then ran validating against the - collected output - -5. Results - * If any failures were detected the test suite returns a failure - * Results can be dumped in yaml format to a specified file using the - ``-r .yaml`` option - -Configuring the Test Suite --------------------------- - -Most of the behavior of the test suite is configurable through several yaml -files. These control the behavior of the test suite's platforms, images, and -tests. The main config files for platforms, images and test cases are -``platforms.yaml``, ``releases.yaml`` and ``testcases.yaml``. - -Config handling -^^^^^^^^^^^^^^^ - -All configurable parts of the test suite use a defaults + overrides system -for managing config entries. All base config items are dictionaries. - -Merging is done on a key-by-key basis, with all keys in the default and -override represented in the final result. If a key exists both in -the defaults and the overrides, then the behavior depends on the type of data -the key refers to. If it is atomic data or a list, then the overrides will -replace the default. If the data is a dictionary then the value will be the -result of merging that dictionary from the default config and that -dictionary from the overrides. - -Merging is done using the function -``tests.cloud_tests.config.merge_config``, which can be examined for more -detail on config merging behavior. - -The following demonstrates merge behavior: - -.. code-block:: yaml - - defaults: - list_item: - - list_entry_1 - - list_entry_2 - int_item_1: 123 - int_item_2: 234 - dict_item: - subkey_1: 1 - subkey_2: 2 - subkey_dict: - subsubkey_1: a - subsubkey_2: b - - overrides: - list_item: - - overridden_list_entry - int_item_1: 0 - dict_item: - subkey_2: false - subkey_dict: - subsubkey_2: 'new value' - - result: - list_item: - - overridden_list_entry - int_item_1: 0 - int_item_2: 234 - dict_item: - subkey_1: 1 - subkey_2: false - subkey_dict: - subsubkey_1: a - subsubkey_2: 'new value' - - -Image Config ------------- - -Image configuration is handled in ``releases.yaml``. The image configuration -controls how platforms locate and acquire images, how the platforms should -interact with the images, how platforms should detect when an image has -fully booted, any options that are required to set the image up, and -features that the image supports. - -Since settings for locating an image and interacting with it differ from -platform to platform, there are 4 levels of settings available for images on -top of the default image settings. The structure of the image config file -is: - -.. code-block:: yaml - - default_release_config: - default: - ... - : - ... - : - ... - - releases: - : - : - ... - : - ... - : - ... - - -The base config is created from the overall defaults and the overrides for -the platform. The overrides are created from the default config for the -image and the platform specific overrides for the image. - -System Boot -^^^^^^^^^^^ - -The test suite must be able to test if a system has fully booted and if -cloud-init has finished running, so that running collect scripts does not -race against the target image booting. This is done using the -``system_ready_script`` and ``cloud_init_ready_script`` image config keys. - -Each of these keys accepts a small bash test statement as a string that must -return 0 or 1. Since this test statement will be added into a larger bash -statement it must be a single statement using the ``[`` test syntax. - -The default image config provides a system ready script that works for any -systemd based image. If the image is not systemd based, then a different -test statement must be provided. The default config also provides a test -for whether or not cloud-init has finished which checks for the file -``/run/cloud-init/result.json``. This should be sufficient for most systems -as writing this file is one of the last things cloud-init does. - -The setting ``boot_timeout`` controls how long, in seconds, the platform -should wait for an image to boot. If the system ready script has not -indicated that the system is fully booted within this time an error will be -raised. - -Feature Flags -^^^^^^^^^^^^^ - -Not all test cases can work on all images due to features the test case -requires not being present on that image. If a test case requires features -in an image that are not likely to be present across all distros and -platforms that the test suite supports, then the test can be skipped -everywhere it is not supported. - -Feature flags, which are names for features supported on some images, but -not all that may be required by test cases. Configuration for feature flags -is provided in ``releases.yaml`` under the ``features`` top level key. The -features config includes a list of all currently defined feature flags, -their meanings, and a list of feature groups. - -Feature groups are groups of features that many images have in common. For -example, the ``Ubuntu_specific`` feature group includes features that -should be present across most Ubuntu releases, but may or may not be for -other distros. Feature groups are specified for an image as a list under -the key ``feature_groups``. - -An image's feature flags are derived from the features groups that that -image has and any feature overrides provided. Feature overrides can be -specified under the ``features`` key which accepts a dictionary of -``{: true/false}`` mappings. If a feature is omitted from an -image's feature flags or set to false in the overrides then the test suite -will skip any tests that require that feature when using that image. - -Feature flags may be overridden at run time using the ``--feature-override`` -command line argument. It accepts a feature flag and value to set in the -format ``=true/false``. Multiple ``--feature-override`` -flags can be used, and will all be applied to all feature flags for images -used during a test. - -Setup Overrides -^^^^^^^^^^^^^^^ - -If an image requires some of the options for image setup to be used, then it -may specify overrides for the command line arguments passed into setup -image. These may be specified as a dictionary under the ``setup_overrides`` -key. When an image is set up, the arguments that control how it is set up -will be the arguments from the command line, with any entries in -``setup_overrides`` used to override these arguments. - -For example, images that do not come with cloud-init already installed -should have ``setup_overrides: {upgrade: true}`` specified so that in the -event that no additional setup options are given, cloud-init will be -installed from the image's repos before running tests. Note that if other -options such as ``--deb`` are passed in on the command line, these will -still work as expected, since apt's policy for cloud-init would prefer the -locally installed deb over an older version from the repos. - -Platform Specific Options -^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are many platform specific options in image configuration that allow -platforms to locate images and that control additional setup that the -platform may have to do to make the image usable. For information on how -these work, please consult the documentation for that platform in the -integration testing suite and the ``releases.yaml`` file for examples. - -Error Handling --------------- - -The test suite makes an attempt to run as many tests as possible even in the -event of some failing so that automated runs collect as much data as -possible. In the event that something goes wrong while setting up for or -running a test, the test suite will attempt to continue running any tests -which have not been affected by the error. - -For example, if the test suite was told to run tests on one platform for two -releases and an error occurred setting up the first image, all tests for -that image would be skipped, and the test suite would continue to set up -the second image and run tests on it. Or, if the system does not start -properly for one test case out of many to run on that image, that test case -will be skipped and the next one will be run. - -Note that if any errors occur, the test suite will record the failure and -where it occurred in the result data and write it out to the specified -result file. - -Results -------- - -The test suite generates result data that includes how long each stage of -the test suite took and which parts were and were not successful. This data -is dumped to the log after the collect and verify stages, and may also be -written out in yaml format to a file. If part of the setup failed, the -traceback for the failure and the error message will be included in the -result file. If a test verifier finds a problem with the collected data -from a test run, the class, test function and test will be recorded in the -result data. - -Exit Codes -^^^^^^^^^^ - -The test suite counts how many errors occur throughout a run. The exit code -after a run is the number of errors that occurred. If the exit code is -non-zero then something is wrong either with the test suite, the -configuration for an image, a test case, or cloud-init itself. - -Note that the exit code does not always directly correspond to the number -of failed test cases, since in some cases, a single error during image setup -can mean that several test cases are not run. If run is used, then the exit -code will be the sum of the number of errors in the collect and verify -stages. - -Data Dir -^^^^^^^^ - -When using run, the collected data is written into a temporary directory. In -the event that all tests pass, this directory is deleted, but if a test -fails or an error occurs, this data will be left in place, and a message -will be written to the log giving the location of the data. diff --git a/integration-requirements.txt b/integration-requirements.txt index 13cfb9d7..64455c79 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -4,6 +4,8 @@ # Note: Changes to this requirements may require updates to # the packages/pkg-deps.json file as well. # +pytest +git+https://github.com/canonical/pycloudlib.git # ec2 backend boto3==1.14.53 diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py new file mode 100644 index 00000000..a170bfc9 --- /dev/null +++ b/tests/integration_tests/conftest.py @@ -0,0 +1,106 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import os +import logging +import pytest +import sys +from contextlib import contextmanager + +from tests.integration_tests import integration_settings +from tests.integration_tests.platforms import ( + dynamic_client, + LxdContainerClient, + client_name_to_class +) + +log = logging.getLogger('integration_testing') +log.addHandler(logging.StreamHandler(sys.stdout)) +log.setLevel(logging.INFO) + + +def pytest_runtest_setup(item): + """Skip tests on unsupported clouds. + + A test can take any number of marks to specify the platforms it can + run on. If a platform(s) is specified and we're not running on that + platform, then skip the test. If platform specific marks are not + specified, then we assume the test can be run anywhere. + """ + all_platforms = client_name_to_class.keys() + supported_platforms = set(all_platforms).intersection( + mark.name for mark in item.iter_markers()) + current_platform = integration_settings.PLATFORM + if supported_platforms and current_platform not in supported_platforms: + pytest.skip('Cannot run on platform {}'.format(current_platform)) + + +# disable_subp_usage is defined at a higher level, but we don't +# want it applied here +@pytest.fixture() +def disable_subp_usage(request): + pass + + +@pytest.fixture(scope='session', autouse=True) +def setup_image(): + """Setup the target environment with the correct version of cloud-init. + + So we can launch instances / run tests with the correct image + """ + client = dynamic_client() + log.info('Setting up environment for %s', client.datasource) + if integration_settings.CLOUD_INIT_SOURCE == 'NONE': + pass # that was easy + elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + if not isinstance(client, LxdContainerClient): + raise ValueError( + 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') + # The mount needs to happen after the instance is launched, so + # no further action needed here + elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED': + client.launch() + client.install_proposed_image() + elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'): + client.launch() + client.install_ppa(integration_settings.CLOUD_INIT_SOURCE) + elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)): + client.launch() + client.install_deb() + if client.instance: + # Even if we're keeping instances, we don't want to keep this + # one around as it was just for image creation + client.destroy() + log.info('Done with environment setup') + + +@contextmanager +def _client(request, fixture_utils): + """Fixture implementation for the client fixtures. + + Launch the dynamic IntegrationClient instance using any provided + userdata, yield to the test, then cleanup + """ + user_data = fixture_utils.closest_marker_first_arg_or( + request, 'user_data', None) + with dynamic_client(user_data=user_data) as instance: + yield instance + + +@pytest.yield_fixture +def client(request, fixture_utils): + """Provide a client that runs for every test.""" + with _client(request, fixture_utils) as client: + yield client + + +@pytest.yield_fixture(scope='module') +def module_client(request, fixture_utils): + """Provide a client that runs once per module.""" + with _client(request, fixture_utils) as client: + yield client + + +@pytest.yield_fixture(scope='class') +def class_client(request, fixture_utils): + """Provide a client that runs once per class.""" + with _client(request, fixture_utils) as client: + yield client diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py new file mode 100644 index 00000000..ddd587db --- /dev/null +++ b/tests/integration_tests/integration_settings.py @@ -0,0 +1,95 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import os + +################################################################## +# LAUNCH SETTINGS +################################################################## + +# Keep instance (mostly for debugging) when test is finished +KEEP_INSTANCE = False + +# One of: +# lxd_container +# ec2 +# gce +# oci +PLATFORM = 'lxd_container' + +# The cloud-specific instance type to run. E.g., a1.medium on AWS +# If the pycloudlib instance provides a default, this can be left None +INSTANCE_TYPE = None + +# Determines the base image to use or generate new images from. +# Can be the name of the OS if running a stock image, +# otherwise the id of the image being used if using a custom image +OS_IMAGE = 'focal' + +# Populate if you want to use a pre-launched instance instead of +# creating a new one. The exact contents will be platform dependent +EXISTING_INSTANCE_ID = None + +################################################################## +# IMAGE GENERATION SETTINGS +################################################################## + +# Depending on where we are in the development / test / SRU cycle, we'll want +# different methods of getting the source code to our SUT. Because of +# this there are a number of different ways to initialize +# the target environment. + +# Can be any of the following: +# NONE +# Don't modify the target environment at all. This will run +# cloud-init with whatever code was baked into the image +# IN_PLACE +# LXD CONTAINER only. Mount the source code as-is directly into +# the container to override the pre-existing cloudinit module. This +# won't work for non-local LXD remotes and won't run any installation +# code. +# PROPOSED +# Install from the Ubuntu proposed repo +# , e.g., ppa:cloud-init-dev/proposed +# Install from a PPA. It MUST start with 'ppa:' +# +# A path to a valid package to be uploaded and installed +CLOUD_INIT_SOURCE = 'NONE' + +################################################################## +# GCE SPECIFIC SETTINGS +################################################################## +# Required for GCE +GCE_PROJECT = None + +# You probably want to override these +GCE_REGION = 'us-central1' +GCE_ZONE = 'a' + +################################################################## +# OCI SPECIFIC SETTINGS +################################################################## +# Compartment-id found at +# https://console.us-phoenix-1.oraclecloud.com/a/identity/compartments +# Required for Oracle +OCI_COMPARTMENT_ID = None + +################################################################## +# USER SETTINGS OVERRIDES +################################################################## +# Bring in any user-file defined settings +try: + from tests.integration_tests.user_settings import * # noqa +except ImportError: + pass + +################################################################## +# ENVIRONMENT SETTINGS OVERRIDES +################################################################## +# Any of the settings in this file can be overridden with an +# environment variable of the same name prepended with CLOUD_INIT_ +# E.g., CLOUD_INIT_PLATFORM +# Perhaps a bit too hacky, but it works :) +current_settings = [var for var in locals() if var.isupper()] +for setting in current_settings: + globals()[setting] = os.getenv( + 'CLOUD_INIT_{}'.format(setting), globals()[setting] + ) diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py new file mode 100644 index 00000000..b42414b9 --- /dev/null +++ b/tests/integration_tests/platforms.py @@ -0,0 +1,235 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from abc import ABC, abstractmethod +import logging +import os +from tempfile import NamedTemporaryFile + +from pycloudlib import EC2, GCE, Azure, OCI, LXD +from pycloudlib.cloud import BaseCloud +from pycloudlib.instance import BaseInstance + +import cloudinit +from cloudinit.subp import subp +from tests.integration_tests import integration_settings + +try: + from typing import Callable, Optional +except ImportError: + pass + + +log = logging.getLogger('integration_testing') + + +class IntegrationClient(ABC): + client = None # type: Optional[BaseCloud] + instance = None # type: Optional[BaseInstance] + datasource = None # type: Optional[str] + use_sudo = True + current_image = None + + def __init__(self, user_data=None, instance_type=None, wait=True, + settings=integration_settings, launch_kwargs=None): + self.user_data = user_data + self.instance_type = settings.INSTANCE_TYPE if \ + instance_type is None else instance_type + self.wait = wait + self.settings = settings + self.launch_kwargs = launch_kwargs if launch_kwargs else {} + self.client = self._get_client() + + @abstractmethod + def _get_client(self): + raise NotImplementedError + + def _get_image(self): + if self.current_image: + return self.current_image + image_id = self.settings.OS_IMAGE + try: + image_id = self.client.released_image(self.settings.OS_IMAGE) + except (ValueError, IndexError): + pass + return image_id + + def launch(self): + if self.settings.EXISTING_INSTANCE_ID: + log.info( + 'Not launching instance due to EXISTING_INSTANCE_ID. ' + 'Instance id: %s', self.settings.EXISTING_INSTANCE_ID) + self.instance = self.client.get_instance( + self.settings.EXISTING_INSTANCE_ID + ) + return + image_id = self._get_image() + launch_args = { + 'image_id': image_id, + 'user_data': self.user_data, + 'wait': self.wait, + } + if self.instance_type: + launch_args['instance_type'] = self.instance_type + launch_args.update(self.launch_kwargs) + self.instance = self.client.launch(**launch_args) + log.info('Launched instance: %s', self.instance) + + def destroy(self): + self.instance.delete() + + def execute(self, command): + return self.instance.execute(command) + + def pull_file(self, remote_file, local_file): + self.instance.pull_file(remote_file, local_file) + + def push_file(self, local_path, remote_path): + self.instance.push_file(local_path, remote_path) + + def read_from_file(self, remote_path) -> str: + tmp_file = NamedTemporaryFile('r') + self.pull_file(remote_path, tmp_file.name) + with tmp_file as f: + contents = f.read() + return contents + + def write_to_file(self, remote_path, contents: str): + # Writes file locally and then pushes it rather + # than writing the file directly on the instance + with NamedTemporaryFile('w', delete=False) as tmp_file: + tmp_file.write(contents) + + try: + self.push_file(tmp_file.name, remote_path) + finally: + os.unlink(tmp_file.name) + + def snapshot(self): + return self.client.snapshot(self.instance, clean=True) + + def _install_new_cloud_init(self, remote_script): + self.execute(remote_script) + version = self.execute('cloud-init -v').split()[-1] + log.info('Installed cloud-init version: %s', version) + self.instance.clean() + image_id = self.snapshot() + log.info('Created new image: %s', image_id) + IntegrationClient.current_image = image_id + + def install_proposed_image(self): + log.info('Installing proposed image') + remote_script = ( + '{sudo} echo deb "http://archive.ubuntu.com/ubuntu ' + '$(lsb_release -sc)-proposed main" | ' + '{sudo} tee /etc/apt/sources.list.d/proposed.list\n' + '{sudo} apt-get update -q\n' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '') + self._install_new_cloud_init(remote_script) + + def install_ppa(self, repo): + log.info('Installing PPA') + remote_script = ( + '{sudo} add-apt-repository {repo} -y && ' + '{sudo} apt-get update -q && ' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '', repo=repo) + self._install_new_cloud_init(remote_script) + + def install_deb(self): + log.info('Installing deb package') + deb_path = integration_settings.CLOUD_INIT_SOURCE + deb_name = os.path.basename(deb_path) + remote_path = '/var/tmp/{}'.format(deb_name) + self.push_file( + local_path=integration_settings.CLOUD_INIT_SOURCE, + remote_path=remote_path) + remote_script = '{sudo} dpkg -i {path}'.format( + sudo='sudo' if self.use_sudo else '', path=remote_path) + self._install_new_cloud_init(remote_script) + + def __enter__(self): + self.launch() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.settings.KEEP_INSTANCE: + self.destroy() + + +class Ec2Client(IntegrationClient): + datasource = 'ec2' + + def _get_client(self): + return EC2(tag='ec2-integration-test') + + +class GceClient(IntegrationClient): + datasource = 'gce' + + def _get_client(self): + return GCE( + tag='gce-integration-test', + project=self.settings.GCE_PROJECT, + region=self.settings.GCE_REGION, + zone=self.settings.GCE_ZONE, + ) + + +class AzureClient(IntegrationClient): + datasource = 'azure' + + def _get_client(self): + return Azure(tag='azure-integration-test') + + +class OciClient(IntegrationClient): + datasource = 'oci' + + def _get_client(self): + return OCI( + tag='oci-integration-test', + compartment_id=self.settings.OCI_COMPARTMENT_ID + ) + + +class LxdContainerClient(IntegrationClient): + datasource = 'lxd_container' + use_sudo = False + + def _get_client(self): + return LXD(tag='lxd-integration-test') + + def _mount_source(self): + command = ( + 'lxc config device add {name} host-cloud-init disk ' + 'source={cloudinit_path} ' + 'path=/usr/lib/python3/dist-packages/cloudinit' + ).format( + name=self.instance.name, cloudinit_path=cloudinit.__path__[0]) + subp(command.split()) + + def launch(self): + super().launch() + if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + self._mount_source() + + +client_name_to_class = { + 'ec2': Ec2Client, + 'gce': GceClient, + # 'azure': AzureClient, # Not supported yet + 'oci': OciClient, + 'lxd_container': LxdContainerClient +} + +try: + dynamic_client = client_name_to_class[ + integration_settings.PLATFORM + ] # type: Callable[..., IntegrationClient] +except KeyError: + raise ValueError( + "{} is an invalid PLATFORM specified in settings. " + "Must be one of {}".format( + integration_settings.PLATFORM, list(client_name_to_class.keys()) + ) + ) diff --git a/tox.ini b/tox.ini index a92c63e0..3bc83a2a 100644 --- a/tox.ini +++ b/tox.ini @@ -139,8 +139,15 @@ deps = [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped +testpaths = cloudinit tests/unittests addopts = --strict markers = allow_subp_for: allow subp usage for the given commands (disable_subp_usage) allow_all_subp: allow all subp usage (disable_subp_usage) ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures + ec2: test will only run on EC2 platform + gce: test will only run on GCE platform + azure: test will only run on Azure platform + oci: test will only run on OCI platform + lxd_container: test will only run in LXD container + user_data: the user data to be passed to the test instance -- cgit v1.2.3 From 11fa714f3a7b031657b642d0c2f0155854d5ee0e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 2 Oct 2020 15:13:19 -0400 Subject: add integration test for LP: #1886531 (#592) --- tests/integration_tests/bugs/test_lp1886531.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/integration_tests/bugs/test_lp1886531.py diff --git a/tests/integration_tests/bugs/test_lp1886531.py b/tests/integration_tests/bugs/test_lp1886531.py new file mode 100644 index 00000000..b1112185 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1886531.py @@ -0,0 +1,27 @@ +"""Integration test for LP: #1886531 + +This test replicates the failure condition (absent /etc/fstab) on all releases +by removing it in a bootcmd; this runs well before the part of cloud-init which +causes the failure. + +The only required assertion is that cloud-init does not emit a WARNING to the +log: this indicates that the fstab parsing code has not failed. + +https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1886531 +""" +import pytest + + +USER_DATA = """\ +#cloud-config +bootcmd: +- rm /etc/fstab +""" + + +class TestLp1886531: + + @pytest.mark.user_data(USER_DATA) + def test_lp1886531(self, client): + log_content = client.read_from_file("/var/log/cloud-init.log") + assert "WARNING" not in log_content -- cgit v1.2.3 From 63313335eccb45df0d8eb43d8ed93d994b946ad2 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 5 Oct 2020 11:40:40 -0400 Subject: PULL_REQUEST_TEMPLATE.md: empty checkboxes need a space (#597) --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 82a0fadf..37f55d26 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,6 @@ setup, and teardown. Scripts used may be attached directly to this PR. --> ## Checklist: - - [] My code follows the process laid out in [the documentation](https://cloudinit.readthedocs.io/en/latest/topics/hacking.html) - - [] I have updated or added any unit tests accordingly - - [] I have updated or added any documentation accordingly \ No newline at end of file + - [ ] My code follows the process laid out in [the documentation](https://cloudinit.readthedocs.io/en/latest/topics/hacking.html) + - [ ] I have updated or added any unit tests accordingly + - [ ] I have updated or added any documentation accordingly -- cgit v1.2.3 From b4f1abf9a2470adf447d8032a56a0ee17947f929 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 5 Oct 2020 13:22:55 -0400 Subject: tox.ini: add integration-tests testenv definition (#595) --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tox.ini b/tox.ini index 3bc83a2a..1160072a 100644 --- a/tox.ini +++ b/tox.ini @@ -137,6 +137,15 @@ passenv = HOME TRAVIS deps = -r{toxinidir}/integration-requirements.txt +[testenv:integration-tests] +basepython = python3 +commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} +passenv = CLOUD_INIT_* +# test-requirements.txt is required for ./conftest.py to import successfully +deps = + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/test-requirements.txt + [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped testpaths = cloudinit tests/unittests -- cgit v1.2.3 From 5435205decf3c8582258d0782dc54aebdd154212 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 6 Oct 2020 10:52:20 -0400 Subject: conftest.py: remove top-level import of httpretty (#599) This means that the integration tests do not need to install test-requirements.txt in order to successfully import `conftest.py`. --- conftest.py | 12 +++++++++++- tox.ini | 2 -- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 76e9000a..459b708a 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,16 @@ +"""Global conftest.py + +This conftest is used for unit tests in ``cloudinit/`` and ``tests/unittests/`` +as well as the integration tests in ``tests/integration_tests/``. + +Any imports that are performed at the top-level here must be installed wherever +any of these tests run: that is to say, they must be listed in +``integration-requirements.txt`` and in ``test-requirements.txt``. +""" import os from unittest import mock import pytest -import httpretty as _httpretty from cloudinit import helpers, subp @@ -156,6 +164,8 @@ def httpretty(): unset http_proxy in os.environ if present (to work around https://github.com/gabrielfalcao/HTTPretty/issues/122). """ + import httpretty as _httpretty + restore_proxy = os.environ.pop("http_proxy", None) _httpretty.HTTPretty.allow_net_connect = False _httpretty.reset() diff --git a/tox.ini b/tox.ini index 1160072a..a8681197 100644 --- a/tox.ini +++ b/tox.ini @@ -141,10 +141,8 @@ deps = basepython = python3 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} passenv = CLOUD_INIT_* -# test-requirements.txt is required for ./conftest.py to import successfully deps = -r{toxinidir}/integration-requirements.txt - -r{toxinidir}/test-requirements.txt [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped -- cgit v1.2.3 From efa4d5be85c596c06cfd4c2613ab010ce54796e8 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 6 Oct 2020 11:12:31 -0400 Subject: integration_tests: improve cloud-init.log assertions (#593) --- tests/integration_tests/conftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index a170bfc9..37197ae3 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -104,3 +104,29 @@ def class_client(request, fixture_utils): """Provide a client that runs once per class.""" with _client(request, fixture_utils) as client: yield client + + +def pytest_assertrepr_compare(op, left, right): + """Custom integration test assertion explanations. + + See + https://docs.pytest.org/en/stable/assert.html#defining-your-own-explanation-for-failed-assertions + for pytest's documentation. + """ + if op == "not in" and isinstance(left, str) and isinstance(right, str): + # This stanza emits an improved assertion message if we're testing for + # the presence of a string within a cloud-init log: it will report only + # the specific lines containing the string (instead of the full log, + # the default behaviour). + potential_log_lines = right.splitlines() + first_line = potential_log_lines[0] + if "DEBUG" in first_line and "Cloud-init" in first_line: + # We are looking at a cloud-init log, so just pick out the relevant + # lines + found_lines = [ + line for line in potential_log_lines if left in line + ] + return [ + '"{}" not in cloud-init.log string; unexpectedly found on' + " these lines:".format(left) + ] + found_lines -- cgit v1.2.3 From 5bf287f430b97860bf746e61b83ff53b834592d0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 6 Oct 2020 11:55:49 -0400 Subject: integration_tests: don't error on cloud-init failure (#596) pycloudlib's default behaviour is to raise an exception if cloud-init fails to run in an instance being launched. For cloud-init testing, we want our test assertions to flag up failures, so we disable this behaviour for instances we launch. --- tests/integration_tests/platforms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py index b42414b9..ba1d4e9f 100644 --- a/tests/integration_tests/platforms.py +++ b/tests/integration_tests/platforms.py @@ -28,12 +28,11 @@ class IntegrationClient(ABC): use_sudo = True current_image = None - def __init__(self, user_data=None, instance_type=None, wait=True, + def __init__(self, user_data=None, instance_type=None, settings=integration_settings, launch_kwargs=None): self.user_data = user_data self.instance_type = settings.INSTANCE_TYPE if \ instance_type is None else instance_type - self.wait = wait self.settings = settings self.launch_kwargs = launch_kwargs if launch_kwargs else {} self.client = self._get_client() @@ -65,12 +64,13 @@ class IntegrationClient(ABC): launch_args = { 'image_id': image_id, 'user_data': self.user_data, - 'wait': self.wait, + 'wait': False, } if self.instance_type: launch_args['instance_type'] = self.instance_type launch_args.update(self.launch_kwargs) self.instance = self.client.launch(**launch_args) + self.instance.wait(raise_on_cloudinit_failure=False) log.info('Launched instance: %s', self.instance) def destroy(self): -- cgit v1.2.3 From 8ec8c3fc63a59b85888a0b52356b784314a1d4cc Mon Sep 17 00:00:00 2001 From: Anh Vo Date: Tue, 13 Oct 2020 15:42:54 -0400 Subject: net: add the ability to blacklist network interfaces based on driver during enumeration of physical network devices (#591) --- cloudinit/distros/networking.py | 6 +++- cloudinit/net/__init__.py | 35 ++++++++++++------- cloudinit/sources/DataSourceAzure.py | 36 +++++++++++++------- tests/unittests/test_datasource/test_azure.py | 49 +++++++++++++++++++-------- 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index 10ed249d..e407fa29 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -22,6 +22,9 @@ class Networking(metaclass=abc.ABCMeta): Hierarchy" in HACKING.rst for full details. """ + def __init__(self): + self.blacklist_drivers = None + def _get_current_rename_info(self) -> dict: return net._get_current_rename_info() @@ -68,7 +71,8 @@ class Networking(metaclass=abc.ABCMeta): return net.get_interfaces() def get_interfaces_by_mac(self) -> dict: - return net.get_interfaces_by_mac() + return net.get_interfaces_by_mac( + blacklist_drivers=self.blacklist_drivers) def get_master(self, devname: DeviceName): return net.get_master(devname) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e233149a..75e79ca8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -746,18 +746,22 @@ def get_ib_interface_hwaddr(ifname, ethernet_format): return mac -def get_interfaces_by_mac(): +def get_interfaces_by_mac(blacklist_drivers=None) -> dict: if util.is_FreeBSD(): - return get_interfaces_by_mac_on_freebsd() + return get_interfaces_by_mac_on_freebsd( + blacklist_drivers=blacklist_drivers) elif util.is_NetBSD(): - return get_interfaces_by_mac_on_netbsd() + return get_interfaces_by_mac_on_netbsd( + blacklist_drivers=blacklist_drivers) elif util.is_OpenBSD(): - return get_interfaces_by_mac_on_openbsd() + return get_interfaces_by_mac_on_openbsd( + blacklist_drivers=blacklist_drivers) else: - return get_interfaces_by_mac_on_linux() + return get_interfaces_by_mac_on_linux( + blacklist_drivers=blacklist_drivers) -def get_interfaces_by_mac_on_freebsd(): +def get_interfaces_by_mac_on_freebsd(blacklist_drivers=None) -> dict(): (out, _) = subp.subp(['ifconfig', '-a', 'ether']) # flatten each interface block in a single line @@ -784,7 +788,7 @@ def get_interfaces_by_mac_on_freebsd(): return results -def get_interfaces_by_mac_on_netbsd(): +def get_interfaces_by_mac_on_netbsd(blacklist_drivers=None) -> dict(): ret = {} re_field_match = ( r"(?P\w+).*address:\s" @@ -800,7 +804,7 @@ def get_interfaces_by_mac_on_netbsd(): return ret -def get_interfaces_by_mac_on_openbsd(): +def get_interfaces_by_mac_on_openbsd(blacklist_drivers=None) -> dict(): ret = {} re_field_match = ( r"(?P\w+).*lladdr\s" @@ -815,12 +819,13 @@ def get_interfaces_by_mac_on_openbsd(): return ret -def get_interfaces_by_mac_on_linux(): +def get_interfaces_by_mac_on_linux(blacklist_drivers=None) -> dict: """Build a dictionary of tuples {mac: name}. Bridges and any devices that have a 'stolen' mac are excluded.""" ret = {} - for name, mac, _driver, _devid in get_interfaces(): + for name, mac, _driver, _devid in get_interfaces( + blacklist_drivers=blacklist_drivers): if mac in ret: raise RuntimeError( "duplicate mac found! both '%s' and '%s' have mac '%s'" % @@ -838,11 +843,13 @@ def get_interfaces_by_mac_on_linux(): return ret -def get_interfaces(): +def get_interfaces(blacklist_drivers=None) -> list: """Return list of interface tuples (name, mac, driver, device_id) Bridges and any devices that have a 'stolen' mac are excluded.""" ret = [] + if blacklist_drivers is None: + blacklist_drivers = [] devs = get_devicelist() # 16 somewhat arbitrarily chosen. Normally a mac is 6 '00:' tokens. zero_mac = ':'.join(('00',) * 16) @@ -866,7 +873,11 @@ def get_interfaces(): # skip nics that have no mac (00:00....) if name != 'lo' and mac == zero_mac[:len(mac)]: continue - ret.append((name, mac, device_driver(name), device_devid(name))) + # skip nics that have drivers blacklisted + driver = device_driver(name) + if driver in blacklist_drivers: + continue + ret.append((name, mac, driver, device_devid(name))) return ret diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 773c60d7..fc32f8b1 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -83,6 +83,25 @@ UBUNTU_EXTENDED_NETWORK_SCRIPTS = [ '/run/network/interfaces.ephemeral.d', ] +# This list is used to blacklist devices that will be considered +# for renaming or fallback interfaces. +# +# On Azure network devices using these drivers are automatically +# configured by the platform and should not be configured by +# cloud-init's network configuration. +# +# Note: +# Azure Dv4 and Ev4 series VMs always have mlx5 hardware. +# https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series +# https://docs.microsoft.com/en-us/azure/virtual-machines/ev4-esv4-series +# Earlier D and E series VMs (such as Dv2, Dv3, and Ev3 series VMs) +# can have either mlx4 or mlx5 hardware, with the older series VMs +# having a higher chance of coming with mlx4 hardware. +# https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series +# https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series +# https://docs.microsoft.com/en-us/azure/virtual-machines/ev3-esv3-series +BLACKLIST_DRIVERS = ['mlx4_core', 'mlx5_core'] + def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid): # extract the 'X' from dev.storvsc.X. if deviceid matches @@ -529,6 +548,8 @@ class DataSourceAzure(sources.DataSource): except Exception as e: LOG.warning("Failed to get system information: %s", e) + self.distro.networking.blacklist_drivers = BLACKLIST_DRIVERS + try: crawled_data = util.log_time( logfunc=LOG.debug, msg='Crawl of metadata service', @@ -1468,23 +1489,12 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @azure_ds_telemetry_reporter def _generate_network_config_from_fallback_config() -> dict: - """Generate fallback network config excluding mlx4_core & mlx5_core devices. + """Generate fallback network config excluding blacklisted devices. @return: Dictionary containing network version 2 standard configuration. """ - # Azure Dv4 and Ev4 series VMs always have mlx5 hardware. - # https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series - # https://docs.microsoft.com/en-us/azure/virtual-machines/ev4-esv4-series - # Earlier D and E series VMs (such as Dv2, Dv3, and Ev3 series VMs) - # can have either mlx4 or mlx5 hardware, with the older series VMs - # having a higher chance of coming with mlx4 hardware. - # https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series - # https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series - # https://docs.microsoft.com/en-us/azure/virtual-machines/ev3-esv3-series - blacklist = ['mlx4_core', 'mlx5_core'] - # generate a network config, blacklist picking mlx4_core and mlx5_core devs return net.generate_fallback_config( - blacklist_drivers=blacklist, config_driver=True) + blacklist_drivers=BLACKLIST_DRIVERS, config_driver=True) @azure_ds_telemetry_reporter diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2b22a879..3b9456f7 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -526,7 +526,7 @@ scbus-1 on xpt0 bus 0 ]) return dsaz - def _get_ds(self, data, agent_command=None, distro=None, + def _get_ds(self, data, agent_command=None, distro='ubuntu', apply_network=None): def dsdevs(): @@ -576,7 +576,7 @@ scbus-1 on xpt0 bus 0 side_effect=_wait_for_files)), ]) - if distro is not None: + if isinstance(distro, str): distro_cls = distros.fetch(distro) distro = distro_cls(distro, data.get('sys_cfg', {}), self.paths) dsrc = dsaz.DataSourceAzure( @@ -638,7 +638,7 @@ scbus-1 on xpt0 bus 0 # Return a non-matching asset tag value m_is_platform_viable.return_value = False dsrc = dsaz.DataSourceAzure( - {}, distro=None, paths=self.paths) + {}, distro=mock.Mock(), paths=self.paths) self.assertFalse(dsrc.get_data()) m_is_platform_viable.assert_called_with(dsrc.seed_dir) @@ -1311,6 +1311,28 @@ scbus-1 on xpt0 bus 0 blacklist_drivers=['mlx4_core', 'mlx5_core'], config_driver=True) + @mock.patch(MOCKPATH + 'net.get_interfaces') + @mock.patch(MOCKPATH + 'util.is_FreeBSD') + def test_blacklist_through_distro( + self, m_is_freebsd, m_net_get_interfaces): + """Verify Azure DS updates blacklist drivers in the distro's + networking object.""" + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + distro_cls = distros.fetch('ubuntu') + distro = distro_cls('ubuntu', {}, self.paths) + dsrc = self._get_ds(data, distro=distro) + dsrc.get_data() + self.assertEqual(distro.networking.blacklist_drivers, + dsaz.BLACKLIST_DRIVERS) + + m_is_freebsd.return_value = False + distro.networking.get_interfaces_by_mac() + m_net_get_interfaces.assert_called_with( + blacklist_drivers=dsaz.BLACKLIST_DRIVERS) + @mock.patch(MOCKPATH + 'subp.subp') def test_get_hostname_with_no_args(self, m_subp): dsaz.get_hostname() @@ -1421,8 +1443,7 @@ class TestAzureBounce(CiTestCase): if ovfcontent is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': ovfcontent}) - dsrc = dsaz.DataSourceAzure( - {}, distro=None, paths=self.paths) + dsrc = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) if agent_command is not None: dsrc.ds_cfg['agent_command'] = agent_command return dsrc @@ -1906,7 +1927,7 @@ class TestClearCachedData(CiTestCase): tmp = self.tmp_dir() paths = helpers.Paths( {'cloud_dir': tmp, 'run_dir': tmp}) - dsrc = dsaz.DataSourceAzure({}, distro=None, paths=paths) + dsrc = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=paths) clean_values = [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds] dsrc.metadata = 'md' dsrc.userdata = 'ud' @@ -1970,7 +1991,7 @@ class TestPreprovisioningShouldReprovision(CiTestCase): """The _should_reprovision method should return true with config flag present.""" isfile.return_value = False - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) self.assertTrue(dsa._should_reprovision( (None, None, {'PreprovisionedVm': True}, None))) @@ -1978,7 +1999,7 @@ class TestPreprovisioningShouldReprovision(CiTestCase): """The _should_reprovision method should return True if the sentinal exists.""" isfile.return_value = True - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) self.assertTrue(dsa._should_reprovision( (None, None, {'preprovisionedvm': False}, None))) @@ -1986,7 +2007,7 @@ class TestPreprovisioningShouldReprovision(CiTestCase): """The _should_reprovision method should return False if config and sentinal are not present.""" isfile.return_value = False - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) self.assertFalse(dsa._should_reprovision((None, None, {}, None))) @mock.patch(MOCKPATH + 'DataSourceAzure._poll_imds') @@ -1997,7 +2018,7 @@ class TestPreprovisioningShouldReprovision(CiTestCase): username = "myuser" odata = {'HostName': hostname, 'UserName': username} _poll_imds.return_value = construct_valid_ovf_env(data=odata) - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) dsa._reprovision() _poll_imds.assert_called_with() @@ -2051,7 +2072,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): fake_resp.side_effect = fake_timeout_once - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): dsa._poll_imds() self.assertEqual(report_ready_func.call_count, 1) @@ -2071,7 +2092,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', 'unknown-245': '624c3620'}] m_media_switch.return_value = None - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): dsa._poll_imds() self.assertEqual(report_ready_func.call_count, 0) @@ -2109,7 +2130,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): full_url = url.format(host) fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf", content="ovf") - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) self.assertTrue(len(dsa._poll_imds()) > 0) self.assertEqual(fake_resp.call_args_list, [mock.call(allow_redirects=True, @@ -2146,7 +2167,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): content = construct_valid_ovf_env(data=odata) fake_resp.return_value = mock.MagicMock(status_code=200, text=content, content=content) - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) md, _ud, cfg, _d = dsa._reprovision() self.assertEqual(md['local-hostname'], hostname) self.assertEqual(cfg['system_info']['default_user']['name'], username) -- cgit v1.2.3 From 3b05b1a6c58dfc7533a16f795405bda0e53aa9d8 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Thu, 15 Oct 2020 07:19:57 -0700 Subject: azure: clean up and refactor report_diagnostic_event (#563) This moves logging into `report_diagnostic_event`, to clean up its callsites. --- cloudinit/sources/DataSourceAzure.py | 120 +++++++++++++++++-------------- cloudinit/sources/helpers/azure.py | 99 ++++++++++++++----------- tests/unittests/test_reporting_hyperv.py | 22 +++++- 3 files changed, 142 insertions(+), 99 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index fc32f8b1..8858fbd5 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -299,9 +299,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): try: set_hostname(temp_hostname, hostname_command) except Exception as e: - msg = 'Failed setting temporary hostname: %s' % e - report_diagnostic_event(msg) - LOG.warning(msg) + report_diagnostic_event( + 'Failed setting temporary hostname: %s' % e, + logger_func=LOG.warning) yield None return try: @@ -356,7 +356,9 @@ class DataSourceAzure(sources.DataSource): cfg=cfg, prev_hostname=previous_hn) except Exception as e: - LOG.warning("Failed publishing hostname: %s", e) + report_diagnostic_event( + "Failed publishing hostname: %s" % e, + logger_func=LOG.warning) util.logexc(LOG, "handling set_hostname failed") return False @@ -454,24 +456,23 @@ class DataSourceAzure(sources.DataSource): except NonAzureDataSource: report_diagnostic_event( - "Did not find Azure data source in %s" % cdev) + "Did not find Azure data source in %s" % cdev, + logger_func=LOG.debug) continue except BrokenAzureDataSource as exc: msg = 'BrokenAzureDataSource: %s' % exc - report_diagnostic_event(msg) + report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) except util.MountFailedError: - msg = '%s was not mountable' % cdev - report_diagnostic_event(msg) - LOG.warning(msg) + report_diagnostic_event( + '%s was not mountable' % cdev, logger_func=LOG.warning) continue perform_reprovision = reprovision or self._should_reprovision(ret) if perform_reprovision: if util.is_FreeBSD(): msg = "Free BSD is not supported for PPS VMs" - LOG.error(msg) - report_diagnostic_event(msg) + report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) ret = self._reprovision() imds_md = get_metadata_from_imds( @@ -486,16 +487,18 @@ class DataSourceAzure(sources.DataSource): 'userdata_raw': userdata_raw}) found = cdev - LOG.debug("found datasource in %s", cdev) + report_diagnostic_event( + 'found datasource in %s' % cdev, logger_func=LOG.debug) break if not found: msg = 'No Azure metadata found' - report_diagnostic_event(msg) + report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) if found == ddir: - LOG.debug("using files cached in %s", ddir) + report_diagnostic_event( + "using files cached in %s" % ddir, logger_func=LOG.debug) seed = _get_random_seed() if seed: @@ -516,7 +519,8 @@ class DataSourceAzure(sources.DataSource): self._report_ready(lease=lease) except Exception as e: report_diagnostic_event( - "exception while reporting ready: %s" % e) + "exception while reporting ready: %s" % e, + logger_func=LOG.error) raise return crawled_data @@ -605,14 +609,14 @@ class DataSourceAzure(sources.DataSource): except KeyError: log_msg = 'Unable to get keys from IMDS, falling back to OVF' LOG.debug(log_msg) - report_diagnostic_event(log_msg) + report_diagnostic_event(log_msg, logger_func=LOG.debug) try: ssh_keys = self.metadata['public-keys'] LOG.debug('Retrieved keys from OVF') except KeyError: log_msg = 'No keys available from OVF' LOG.debug(log_msg) - report_diagnostic_event(log_msg) + report_diagnostic_event(log_msg, logger_func=LOG.debug) return ssh_keys @@ -666,16 +670,14 @@ class DataSourceAzure(sources.DataSource): if self.imds_poll_counter == self.imds_logging_threshold: # Reducing the logging frequency as we are polling IMDS self.imds_logging_threshold *= 2 - LOG.debug("Call to IMDS with arguments %s failed " - "with status code %s after %s retries", - msg, exception.code, self.imds_poll_counter) LOG.debug("Backing off logging threshold for the same " "exception to %d", self.imds_logging_threshold) report_diagnostic_event("poll IMDS with %s failed. " "Exception: %s and code: %s" % (msg, exception.cause, - exception.code)) + exception.code), + logger_func=LOG.debug) self.imds_poll_counter += 1 return True else: @@ -684,12 +686,15 @@ class DataSourceAzure(sources.DataSource): report_diagnostic_event("poll IMDS with %s failed. " "Exception: %s and code: %s" % (msg, exception.cause, - exception.code)) + exception.code), + logger_func=LOG.warning) return False - LOG.debug("poll IMDS failed with an unexpected exception: %s", - exception) - return False + report_diagnostic_event( + "poll IMDS failed with an " + "unexpected exception: %s" % exception, + logger_func=LOG.warning) + return False LOG.debug("Wait for vnetswitch to happen") while True: @@ -709,8 +714,9 @@ class DataSourceAzure(sources.DataSource): try: nl_sock = netlink.create_bound_netlink_socket() except netlink.NetlinkCreateSocketError as e: - report_diagnostic_event(e) - LOG.warning(e) + report_diagnostic_event( + 'Failed to create bound netlink socket: %s' % e, + logger_func=LOG.warning) self._ephemeral_dhcp_ctx.clean_network() break @@ -729,9 +735,10 @@ class DataSourceAzure(sources.DataSource): try: netlink.wait_for_media_disconnect_connect( nl_sock, lease['interface']) - except AssertionError as error: - report_diagnostic_event(error) - LOG.error(error) + except AssertionError as e: + report_diagnostic_event( + 'Error while waiting for vnet switch: %s' % e, + logger_func=LOG.error) break vnet_switched = True @@ -757,9 +764,11 @@ class DataSourceAzure(sources.DataSource): if vnet_switched: report_diagnostic_event("attempted dhcp %d times after reuse" % - dhcp_attempts) + dhcp_attempts, + logger_func=LOG.debug) report_diagnostic_event("polled imds %d times after reuse" % - self.imds_poll_counter) + self.imds_poll_counter, + logger_func=LOG.debug) return return_val @@ -768,10 +777,10 @@ class DataSourceAzure(sources.DataSource): """Tells the fabric provisioning has completed """ try: get_metadata_from_fabric(None, lease['unknown-245']) - except Exception: - LOG.warning( - "Error communicating with Azure fabric; You may experience." - "connectivity issues.", exc_info=True) + except Exception as e: + report_diagnostic_event( + "Error communicating with Azure fabric; You may experience " + "connectivity issues: %s" % e, logger_func=LOG.warning) def _should_reprovision(self, ret): """Whether or not we should poll IMDS for reprovisioning data. @@ -849,10 +858,7 @@ class DataSourceAzure(sources.DataSource): except Exception as e: report_diagnostic_event( "Error communicating with Azure fabric; You may experience " - "connectivity issues: %s" % e) - LOG.warning( - "Error communicating with Azure fabric; You may experience " - "connectivity issues.", exc_info=True) + "connectivity issues: %s" % e, logger_func=LOG.warning) return False util.del_file(REPORTED_READY_MARKER_FILE) @@ -1017,9 +1023,10 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, log_pre="Azure ephemeral disk: ") if missing: - LOG.warning("ephemeral device '%s' did" - " not appear after %d seconds.", - devpath, maxwait) + report_diagnostic_event( + "ephemeral device '%s' did not appear after %d seconds." % + (devpath, maxwait), + logger_func=LOG.warning) return result = False @@ -1104,7 +1111,9 @@ def pubkeys_from_crt_files(flist): errors.append(fname) if errors: - LOG.warning("failed to convert the crt files to pubkey: %s", errors) + report_diagnostic_event( + "failed to convert the crt files to pubkey: %s" % errors, + logger_func=LOG.warning) return pubkeys @@ -1216,7 +1225,7 @@ def read_azure_ovf(contents): dom = minidom.parseString(contents) except Exception as e: error_str = "Invalid ovf-env.xml: %s" % e - report_diagnostic_event(error_str) + report_diagnostic_event(error_str, logger_func=LOG.warning) raise BrokenAzureDataSource(error_str) from e results = find_child(dom.documentElement, @@ -1523,7 +1532,9 @@ def get_metadata_from_imds(fallback_nic, retries): azure_ds_reporter, fallback_nic): return util.log_time(**kwargs) except Exception as e: - report_diagnostic_event("exception while getting metadata: %s" % e) + report_diagnostic_event( + "exception while getting metadata: %s" % e, + logger_func=LOG.warning) raise @@ -1537,9 +1548,10 @@ def _get_metadata_from_imds(retries): url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers, retries=retries, exception_cb=retry_on_url_exc) except Exception as e: - msg = 'Ignoring IMDS instance metadata: %s' % e - report_diagnostic_event(msg) - LOG.debug(msg) + report_diagnostic_event( + 'Ignoring IMDS instance metadata. ' + 'Get metadata from IMDS failed: %s' % e, + logger_func=LOG.warning) return {} try: from json.decoder import JSONDecodeError @@ -1550,9 +1562,10 @@ def _get_metadata_from_imds(retries): try: return util.load_json(str(response)) except json_decode_error as e: - report_diagnostic_event('non-json imds response' % e) - LOG.warning( - 'Ignoring non-json IMDS instance metadata: %s', str(response)) + report_diagnostic_event( + 'Ignoring non-json IMDS instance metadata response: %s. ' + 'Loading non-json IMDS response failed: %s' % (str(response), e), + logger_func=LOG.warning) return {} @@ -1604,9 +1617,8 @@ def _is_platform_viable(seed_dir): if asset_tag == AZURE_CHASSIS_ASSET_TAG: return True msg = "Non-Azure DMI asset tag '%s' discovered." % asset_tag - LOG.debug(msg) evt.description = msg - report_diagnostic_event(msg) + report_diagnostic_event(msg, logger_func=LOG.debug) if os.path.exists(os.path.join(seed_dir, 'ovf-env.xml')): return True return False diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 79445a81..560cadba 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -180,12 +180,15 @@ def get_system_info(): return evt -def report_diagnostic_event(str): +def report_diagnostic_event( + msg: str, *, logger_func=None) -> events.ReportingEvent: """Report a diagnostic event""" + if callable(logger_func): + logger_func(msg) evt = events.ReportingEvent( DIAGNOSTIC_EVENT_TYPE, 'diagnostic message', - str, events.DEFAULT_EVENT_ORIGIN) - events.report_event(evt) + msg, events.DEFAULT_EVENT_ORIGIN) + events.report_event(evt, excluded_handler_types={"log"}) # return the event for unit testing purpose return evt @@ -215,7 +218,8 @@ def push_log_to_kvp(file_name=CFG_BUILTIN['def_log_file']): log_pushed_to_kvp = bool(os.path.isfile(LOG_PUSHED_TO_KVP_MARKER_FILE)) if log_pushed_to_kvp: - report_diagnostic_event("cloud-init.log is already pushed to KVP") + report_diagnostic_event( + "cloud-init.log is already pushed to KVP", logger_func=LOG.debug) return LOG.debug("Dumping cloud-init.log file to KVP") @@ -225,13 +229,15 @@ def push_log_to_kvp(file_name=CFG_BUILTIN['def_log_file']): seek_index = max(f.tell() - MAX_LOG_TO_KVP_LENGTH, 0) report_diagnostic_event( "Dumping last {} bytes of cloud-init.log file to KVP".format( - f.tell() - seek_index)) + f.tell() - seek_index), + logger_func=LOG.debug) f.seek(seek_index, os.SEEK_SET) report_compressed_event("cloud-init.log", f.read()) util.write_file(LOG_PUSHED_TO_KVP_MARKER_FILE, '') except Exception as ex: - report_diagnostic_event("Exception when dumping log file: %s" % - repr(ex)) + report_diagnostic_event( + "Exception when dumping log file: %s" % repr(ex), + logger_func=LOG.warning) @contextmanager @@ -305,9 +311,9 @@ class GoalState: try: self.root = ElementTree.fromstring(unparsed_xml) except ElementTree.ParseError as e: - msg = 'Failed to parse GoalState XML: %s' - LOG.warning(msg, e) - report_diagnostic_event(msg % (e,)) + report_diagnostic_event( + 'Failed to parse GoalState XML: %s' % e, + logger_func=LOG.warning) raise self.container_id = self._text_from_xpath('./Container/ContainerId') @@ -317,9 +323,8 @@ class GoalState: for attr in ("container_id", "instance_id", "incarnation"): if getattr(self, attr) is None: - msg = 'Missing %s in GoalState XML' - LOG.warning(msg, attr) - report_diagnostic_event(msg % (attr,)) + msg = 'Missing %s in GoalState XML' % attr + report_diagnostic_event(msg, logger_func=LOG.warning) raise InvalidGoalStateXMLException(msg) self.certificates_xml = None @@ -513,9 +518,9 @@ class GoalStateHealthReporter: try: self._post_health_report(document=document) except Exception as e: - msg = "exception while reporting ready: %s" % e - LOG.error(msg) - report_diagnostic_event(msg) + report_diagnostic_event( + "exception while reporting ready: %s" % e, + logger_func=LOG.error) raise LOG.info('Reported ready to Azure fabric.') @@ -698,39 +703,48 @@ class WALinuxAgentShim: value = dhcp245 LOG.debug("Using Azure Endpoint from dhcp options") if value is None: - report_diagnostic_event("No Azure endpoint from dhcp options") - LOG.debug('Finding Azure endpoint from networkd...') + report_diagnostic_event( + 'No Azure endpoint from dhcp options. ' + 'Finding Azure endpoint from networkd...', + logger_func=LOG.debug) value = WALinuxAgentShim._networkd_get_value_from_leases() if value is None: # Option-245 stored in /run/cloud-init/dhclient.hooks/.json # a dhclient exit hook that calls cloud-init-dhclient-hook - report_diagnostic_event("No Azure endpoint from networkd") - LOG.debug('Finding Azure endpoint from hook json...') + report_diagnostic_event( + 'No Azure endpoint from networkd. ' + 'Finding Azure endpoint from hook json...', + logger_func=LOG.debug) dhcp_options = WALinuxAgentShim._load_dhclient_json() value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: # Fallback and check the leases file if unsuccessful - report_diagnostic_event("No Azure endpoint from dhclient logs") - LOG.debug("Unable to find endpoint in dhclient logs. " - " Falling back to check lease files") + report_diagnostic_event( + 'No Azure endpoint from dhclient logs. ' + 'Unable to find endpoint in dhclient logs. ' + 'Falling back to check lease files', + logger_func=LOG.debug) if fallback_lease_file is None: - LOG.warning("No fallback lease file was specified.") + report_diagnostic_event( + 'No fallback lease file was specified.', + logger_func=LOG.warning) value = None else: - LOG.debug("Looking for endpoint in lease file %s", - fallback_lease_file) + report_diagnostic_event( + 'Looking for endpoint in lease file %s' + % fallback_lease_file, logger_func=LOG.debug) value = WALinuxAgentShim._get_value_from_leases_file( fallback_lease_file) if value is None: - msg = "No lease found; using default endpoint" - report_diagnostic_event(msg) - LOG.warning(msg) value = DEFAULT_WIRESERVER_ENDPOINT + report_diagnostic_event( + 'No lease found; using default endpoint: %s' % value, + logger_func=LOG.warning) endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) - msg = 'Azure endpoint found at %s' % endpoint_ip_address - report_diagnostic_event(msg) - LOG.debug(msg) + report_diagnostic_event( + 'Azure endpoint found at %s' % endpoint_ip_address, + logger_func=LOG.debug) return endpoint_ip_address @azure_ds_telemetry_reporter @@ -795,9 +809,9 @@ class WALinuxAgentShim: parent=azure_ds_reporter): response = self.azure_endpoint_client.get(url) except Exception as e: - msg = 'failed to register with Azure: %s' % e - LOG.warning(msg) - report_diagnostic_event(msg) + report_diagnostic_event( + 'failed to register with Azure and fetch GoalState XML: %s' + % e, logger_func=LOG.warning) raise LOG.debug('Successfully fetched GoalState XML.') return response.contents @@ -819,16 +833,15 @@ class WALinuxAgentShim: need_certificate ) except Exception as e: - msg = 'Error processing GoalState XML: %s' % e - LOG.warning(msg) - report_diagnostic_event(msg) + report_diagnostic_event( + 'Error processing GoalState XML: %s' % e, + logger_func=LOG.warning) raise msg = ', '.join([ 'GoalState XML container id: %s' % goal_state.container_id, 'GoalState XML instance id: %s' % goal_state.instance_id, 'GoalState XML incarnation: %s' % goal_state.incarnation]) - LOG.debug(msg) - report_diagnostic_event(msg) + report_diagnostic_event(msg, logger_func=LOG.debug) return goal_state @azure_ds_telemetry_reporter @@ -910,8 +923,10 @@ def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, def dhcp_log_cb(out, err): - report_diagnostic_event("dhclient output stream: %s" % out) - report_diagnostic_event("dhclient error stream: %s" % err) + report_diagnostic_event( + "dhclient output stream: %s" % out, logger_func=LOG.debug) + report_diagnostic_event( + "dhclient error stream: %s" % err, logger_func=LOG.debug) class EphemeralDHCPv4WithReporting: diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index 47ede670..3f63a60e 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -188,18 +188,34 @@ class TextKvpReporter(CiTestCase): if not re.search("variant=" + pattern, evt_msg): raise AssertionError("missing distro variant string") - def test_report_diagnostic_event(self): + def test_report_diagnostic_event_without_logger_func(self): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) + diagnostic_msg = "test_diagnostic" + reporter.publish_event( + azure.report_diagnostic_event(diagnostic_msg)) + reporter.q.join() + kvps = list(reporter._iterate_kvps(0)) + self.assertEqual(1, len(kvps)) + evt_msg = kvps[0]['value'] + + if diagnostic_msg not in evt_msg: + raise AssertionError("missing expected diagnostic message") + def test_report_diagnostic_event_with_logger_func(self): + reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) + logger_func = mock.MagicMock() + diagnostic_msg = "test_diagnostic" reporter.publish_event( - azure.report_diagnostic_event("test_diagnostic")) + azure.report_diagnostic_event(diagnostic_msg, + logger_func=logger_func)) reporter.q.join() kvps = list(reporter._iterate_kvps(0)) self.assertEqual(1, len(kvps)) evt_msg = kvps[0]['value'] - if "test_diagnostic" not in evt_msg: + if diagnostic_msg not in evt_msg: raise AssertionError("missing expected diagnostic message") + logger_func.assert_called_once_with(diagnostic_msg) def test_report_compressed_event(self): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) -- cgit v1.2.3 From d76d6e6749315634efe0494501270740e8fef206 Mon Sep 17 00:00:00 2001 From: Adrian Vladu Date: Thu, 15 Oct 2020 22:39:09 +0300 Subject: openstack: consider product_name as valid chassis tag (#580) Consider valid product names as valid chassis asset tags when detecting OpenStack platform before crawling for OpenStack metadata. As `ds-identify` tool uses product name as valid chassis asset tags, let's replicate the behaviour in the OpenStack platform detection too. This change should be backwards compatible and a temporary fix for the current limitations on the OpenStack platform detection. LP: #1895976 --- cloudinit/sources/DataSourceOpenStack.py | 3 ++- tests/unittests/test_datasource/test_openstack.py | 30 +++++++++++++++++++++++ tools/.github-cla-signers | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index d4b43f44..0ede0a0e 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -32,7 +32,8 @@ DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud' # See github.com/sapcc/helm-charts/blob/master/openstack/nova/values.yaml # -> compute.defaults.vmware.smbios_asset_tag for this value DMI_ASSET_TAG_SAPCCLOUD = 'SAP CCloud VM' -VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_SAPCCLOUD] +VALID_DMI_ASSET_TAGS = VALID_DMI_PRODUCT_NAMES +VALID_DMI_ASSET_TAGS += [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_SAPCCLOUD] class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3cfba74d..9b0c1b8a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -548,6 +548,36 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(accept_oracle=False), 'Expected detect_openstack == False.') + def _test_detect_openstack_nova_compute_chassis_asset_tag(self, m_dmi, + m_is_x86, + chassis_tag): + """Return True on OpenStack reporting generic asset-tag.""" + m_is_x86.return_value = True + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'Generic OpenStack Platform' + if dmi_key == 'chassis-asset-tag': + return chassis_tag + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertTrue( + ds.detect_openstack(), + 'Expected detect_openstack == True on Generic OpenStack Platform') + + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_nova_chassis_asset_tag(self, m_dmi, + m_is_x86): + self._test_detect_openstack_nova_compute_chassis_asset_tag( + m_dmi, m_is_x86, 'OpenStack Nova') + + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_compute_chassis_asset_tag(self, m_dmi, + m_is_x86): + self._test_detect_openstack_nova_compute_chassis_asset_tag( + m_dmi, m_is_x86, 'OpenStack Compute') + @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env') @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env, diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index f01e9b66..d93d0153 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,3 +1,4 @@ +ader1990 AlexBaranowski beezly bipinbachhao -- cgit v1.2.3 From 6bd2cda1d3812750027590fe02a5b9415c8f8c9e Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 15 Oct 2020 15:48:38 -0500 Subject: Add Azure support to integration test framework (#604) --- tests/integration_tests/integration_settings.py | 1 + tests/integration_tests/platforms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index ddd587db..a0609f7e 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -10,6 +10,7 @@ KEEP_INSTANCE = False # One of: # lxd_container +# azure # ec2 # gce # oci diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py index ba1d4e9f..3755eb1e 100644 --- a/tests/integration_tests/platforms.py +++ b/tests/integration_tests/platforms.py @@ -217,7 +217,7 @@ class LxdContainerClient(IntegrationClient): client_name_to_class = { 'ec2': Ec2Client, 'gce': GceClient, - # 'azure': AzureClient, # Not supported yet + 'azure': AzureClient, 'oci': OciClient, 'lxd_container': LxdContainerClient } -- cgit v1.2.3 From 268fefa309c20181f18ce1081e26aa90cc0b2f85 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 16 Oct 2020 10:22:50 -0400 Subject: integration_tests: implement citest tests run in Travis (#605) Specifically: * `apt_configure_sources_list` * `ntp_servers` * `set_password_list` * `users_groups` Although not currently run in Travis, `set_password_list_string` was ported over alongside `set_password_list` (as `test_set_password`). --- .../modules/test_apt_configure_sources_list.py | 50 +++++++ .../integration_tests/modules/test_ntp_servers.py | 57 ++++++++ .../integration_tests/modules/test_set_password.py | 149 +++++++++++++++++++++ .../integration_tests/modules/test_users_groups.py | 79 +++++++++++ 4 files changed, 335 insertions(+) create mode 100644 tests/integration_tests/modules/test_apt_configure_sources_list.py create mode 100644 tests/integration_tests/modules/test_ntp_servers.py create mode 100644 tests/integration_tests/modules/test_set_password.py create mode 100644 tests/integration_tests/modules/test_users_groups.py diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py new file mode 100644 index 00000000..d64b3956 --- /dev/null +++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py @@ -0,0 +1,50 @@ +"""Integration test for the apt module's ``sources_list`` functionality. + +This test specifies a ``sources_list`` and then checks that (a) the expected +number of sources.list entries is present, and (b) that each expected line +appears in the file. + +(This is ported from +``tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml``.)""" +import re + +import pytest + + +USER_DATA = """\ +#cloud-config +apt: + primary: + - arches: [default] + uri: http://archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb-src $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + deb-src $SECURITY $RELEASE-security multiverse +""" + +EXPECTED_REGEXES = [ + r"deb http://archive.ubuntu.com/ubuntu [a-z].* main restricted", + r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* main restricted", + r"deb http://archive.ubuntu.com/ubuntu [a-z].* universe restricted", + r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* universe restricted", + r"deb http://security.ubuntu.com/ubuntu [a-z].*security multiverse", + r"deb-src http://security.ubuntu.com/ubuntu [a-z].*security multiverse", +] + + +class TestAptConfigureSourcesList: + + @pytest.mark.user_data(USER_DATA) + def test_sources_list(self, client): + sources_list = client.read_from_file("/etc/apt/sources.list") + assert 6 == len(sources_list.rstrip().split('\n')) + + for expected_re in EXPECTED_REGEXES: + assert re.search(expected_re, sources_list) is not None diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py new file mode 100644 index 00000000..4cad8926 --- /dev/null +++ b/tests/integration_tests/modules/test_ntp_servers.py @@ -0,0 +1,57 @@ +"""Integration test for the ntp module's ``servers`` functionality with ntp. + +This test specifies the use of the `ntp` NTP client, and ensures that the given +NTP servers are configured as expected. + +(This is ported from ``tests/cloud_tests/testcases/modules/ntp_servers.yaml``.) +""" +import re + +import yaml +import pytest + +USER_DATA = """\ +#cloud-config +ntp: + ntp_client: ntp + servers: + - 172.16.15.14 + - 172.16.17.18 +""" + +EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"] + + +@pytest.mark.user_data(USER_DATA) +class TestNtpServers: + + def test_ntp_installed(self, class_client): + """Test that `ntpd --version` succeeds, indicating installation.""" + result = class_client.execute("ntpd --version") + assert 0 == result.return_code + + def test_dist_config_file_is_empty(self, class_client): + """Test that the distributed config file is empty. + + (This test is skipped on all currently supported Ubuntu releases, so + may not actually be needed any longer.) + """ + if class_client.execute("test -e /etc/ntp.conf.dist").failed: + pytest.skip("/etc/ntp.conf.dist does not exist") + dist_file = class_client.read_from_file("/etc/ntp.conf.dist") + assert 0 == len(dist_file.strip().splitlines()) + + def test_ntp_entries(self, class_client): + ntp_conf = class_client.read_from_file("/etc/ntp.conf") + for expected_server in EXPECTED_SERVERS: + assert re.search( + r"^server {} iburst".format(expected_server), + ntp_conf, + re.MULTILINE + ) + + def test_ntpq_servers(self, class_client): + result = class_client.execute("ntpq -p -w -n") + assert result.ok + for expected_server in EXPECTED_SERVERS: + assert expected_server in result.stdout diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py new file mode 100644 index 00000000..ae6fdefc --- /dev/null +++ b/tests/integration_tests/modules/test_set_password.py @@ -0,0 +1,149 @@ +"""Integration test for the set_password module. + +This test specifies a combination of user/password pairs, and ensures that the +system has the correct passwords set. + +There are two tests run here: one tests chpasswd's list being a YAML list, the +other tests chpasswd's list being a string. Both expect the same results, so +they use a mixin to share their test definitions, because we can (of course) +only specify one user-data per instance. +""" +import crypt + +import pytest +import yaml + + +COMMON_USER_DATA = """\ +#cloud-config +ssh_pwauth: yes +users: + - default + - name: tom + # md5 gotomgo + passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" + lock_passwd: false + - name: dick + # md5 gocubsgo + passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" + lock_passwd: false + - name: harry + # sha512 goharrygo + passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3y\ +Uh69tP4GSrGW5XKHxMLiKowJgm/" + lock_passwd: false + - name: jane + # sha256 gojanego + passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." + lock_passwd: false + - name: "mikey" + lock_passwd: false +""" + +LIST_USER_DATA = COMMON_USER_DATA + """ +chpasswd: + list: + - tom:mypassword123! + - dick:RANDOM + - harry:RANDOM + - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +""" + +STRING_USER_DATA = COMMON_USER_DATA + """ +chpasswd: + list: | + tom:mypassword123! + dick:RANDOM + harry:RANDOM + mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +""" + +USERS_DICTS = yaml.safe_load(COMMON_USER_DATA)["users"] +USERS_PASSWD_VALUES = { + user_dict["name"]: user_dict["passwd"] + for user_dict in USERS_DICTS + if "name" in user_dict and "passwd" in user_dict +} + + +class Mixin: + """Shared test definitions.""" + + def _fetch_and_parse_etc_shadow(self, class_client): + """Fetch /etc/shadow and parse it into Python data structures + + Returns: ({user: password}, [duplicate, users]) + """ + shadow_content = class_client.read_from_file("/etc/shadow") + users = {} + dupes = [] + for line in shadow_content.splitlines(): + user, encpw = line.split(":")[0:2] + if user in users: + dupes.append(user) + users[user] = encpw + return users, dupes + + def test_no_duplicate_users_in_shadow(self, class_client): + """Confirm that set_passwords has not added duplicate shadow entries""" + _, dupes = self._fetch_and_parse_etc_shadow(class_client) + + assert [] == dupes + + def test_password_in_users_dict_set_correctly(self, class_client): + """Test that the password specified in the users dict is set.""" + shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + assert USERS_PASSWD_VALUES["jane"] == shadow_users["jane"] + + def test_password_in_chpasswd_list_set_correctly(self, class_client): + """Test that a chpasswd password overrides one in the users dict.""" + shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + mikey_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" + assert mikey_hash == shadow_users["mikey"] + + def test_random_passwords_set_correctly(self, class_client): + """Test that RANDOM chpasswd entries replace users dict passwords.""" + shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + + # These should have been changed + assert shadow_users["harry"] != USERS_PASSWD_VALUES["harry"] + assert shadow_users["dick"] != USERS_PASSWD_VALUES["dick"] + + # To random passwords + assert shadow_users["harry"].startswith("$") + assert shadow_users["dick"].startswith("$") + + # Which are not the same + assert shadow_users["harry"] != shadow_users["dick"] + + def test_explicit_password_set_correctly(self, class_client): + """Test that an explicitly-specified password is set correctly.""" + shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + + fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0] + expected_value = crypt.crypt("mypassword123!", fmt_and_salt) + + assert expected_value == shadow_users["tom"] + + def test_shadow_expected_users(self, class_client): + """Test that the right set of users is in /etc/shadow.""" + shadow = class_client.read_from_file("/etc/shadow") + for user_dict in USERS_DICTS: + if "name" in user_dict: + assert "{}:".format(user_dict["name"]) in shadow + + def test_sshd_config(self, class_client): + """Test that SSH password auth is enabled.""" + sshd_config = class_client.read_from_file("/etc/ssh/sshd_config") + # We look for the exact line match, to avoid a commented line matching + assert "PasswordAuthentication yes" in sshd_config.splitlines() + + +@pytest.mark.user_data(LIST_USER_DATA) +class TestPasswordList(Mixin): + """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass.""" + + +@pytest.mark.user_data(STRING_USER_DATA) +class TestPasswordListString(Mixin): + """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass.""" diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py new file mode 100644 index 00000000..b1fa8c22 --- /dev/null +++ b/tests/integration_tests/modules/test_users_groups.py @@ -0,0 +1,79 @@ +"""Integration test for the user_groups module. + +This test specifies a number of users and groups via user-data, and confirms +that they have been configured correctly in the system under test. +""" +import re + +import pytest + + +USER_DATA = """\ +#cloud-config +# Add groups to the system +groups: + - secret: [root] + - cloud-users + +# Add users to the system. Users are added after groups are added. +users: + - default + - name: foobar + gecos: Foo B. Bar + primary_group: foobar + groups: users + expiredate: 2038-01-19 + lock_passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYe\ +AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: [cloud-users, secret] + lock_passwd: true + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true +""" + + +@pytest.mark.user_data(USER_DATA) +class TestUsersGroups: + @pytest.mark.parametrize( + "getent_args,regex", + [ + # Test the ubuntu group + (["group", "ubuntu"], r"ubuntu:x:[0-9]{4}:"), + # Test the cloud-users group + (["group", "cloud-users"], r"cloud-users:x:[0-9]{4}:barfoo"), + # Test the ubuntu user + ( + ["passwd", "ubuntu"], + r"ubuntu:x:[0-9]{4}:[0-9]{4}:Ubuntu:/home/ubuntu:/bin/bash", + ), + # Test the foobar user + ( + ["passwd", "foobar"], + r"foobar:x:[0-9]{4}:[0-9]{4}:Foo B. Bar:/home/foobar:", + ), + # Test the barfoo user + ( + ["passwd", "barfoo"], + r"barfoo:x:[0-9]{4}:[0-9]{4}:Bar B. Foo:/home/barfoo:", + ), + # Test the cloudy user + (["passwd", "cloudy"], r"cloudy:x:[0-9]{3,4}:"), + ], + ) + def test_users_groups(self, regex, getent_args, class_client): + """Use getent to interrogate the various expected outcomes""" + result = class_client.execute(["getent"] + getent_args) + assert re.search(regex, result.stdout) is not None + + def test_user_root_in_secret(self, class_client): + """Test root user is in 'secret' group.""" + output = class_client.execute("groups root").stdout + _, groups_str = output.split(":", maxsplit=1) + groups = groups_str.split() + assert "secret" in groups -- cgit v1.2.3 From c0e8480678e3a9173c9de1271f651fb3ba375f22 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 16 Oct 2020 10:41:21 -0400 Subject: integration_tests: emit settings to log during setup (#601) --- tests/integration_tests/conftest.py | 1 + tests/integration_tests/platforms.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 37197ae3..9c4eb03a 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -48,6 +48,7 @@ def setup_image(): """ client = dynamic_client() log.info('Setting up environment for %s', client.datasource) + client.emit_settings_to_log() if integration_settings.CLOUD_INIT_SOURCE == 'NONE': pass # that was easy elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py index 3755eb1e..bade5fe0 100644 --- a/tests/integration_tests/platforms.py +++ b/tests/integration_tests/platforms.py @@ -37,6 +37,17 @@ class IntegrationClient(ABC): self.launch_kwargs = launch_kwargs if launch_kwargs else {} self.client = self._get_client() + def emit_settings_to_log(self) -> None: + log.info( + "\n".join( + ["Settings:"] + + [ + "{}={}".format(key, getattr(self.settings, key)) + for key in sorted(self.settings.current_settings) + ] + ) + ) + @abstractmethod def _get_client(self): raise NotImplementedError -- cgit v1.2.3 From 8766784f4b1d1f9f6a9094e1268e4accb811ea7f Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Fri, 16 Oct 2020 08:54:38 -0700 Subject: DataSourceAzure: write marker file after report ready in preprovisioning (#590) DataSourceAzure previously writes the preprovisioning reported ready marker file before it goes through the report ready workflow. On certain VM instances, the marker file is successfully written but then reporting ready fails. Upon rare VM reboots by the platform, cloud-init sees that the report ready marker file already exists. The existence of this marker file tells cloud-init not to report ready again (because it mistakenly assumes that it already reported ready in preprovisioning). In this scenario, cloud-init instead erroneously takes the reprovisioning workflow instead of reporting ready again. --- cloudinit/sources/DataSourceAzure.py | 23 +++++- tests/unittests/test_datasource/test_azure.py | 86 +++++++++++++++++----- .../unittests/test_datasource/test_azure_helper.py | 3 +- 3 files changed, 90 insertions(+), 22 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 8858fbd5..70e32f46 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -720,12 +720,23 @@ class DataSourceAzure(sources.DataSource): self._ephemeral_dhcp_ctx.clean_network() break + report_ready_succeeded = self._report_ready(lease=lease) + if not report_ready_succeeded: + msg = ('Failed reporting ready while in ' + 'the preprovisioning pool.') + report_diagnostic_event(msg, logger_func=LOG.error) + self._ephemeral_dhcp_ctx.clean_network() + raise sources.InvalidMetaDataException(msg) + path = REPORTED_READY_MARKER_FILE LOG.info( "Creating a marker file to report ready: %s", path) util.write_file(path, "{pid}: {time}\n".format( pid=os.getpid(), time=time())) - self._report_ready(lease=lease) + report_diagnostic_event( + 'Successfully created reported ready marker file ' + 'while in the preprovisioning pool.', + logger_func=LOG.debug) report_ready = False with events.ReportEventStack( @@ -773,14 +784,20 @@ class DataSourceAzure(sources.DataSource): return return_val @azure_ds_telemetry_reporter - def _report_ready(self, lease): - """Tells the fabric provisioning has completed """ + def _report_ready(self, lease: dict) -> bool: + """Tells the fabric provisioning has completed. + + @param lease: dhcp lease to use for sending the ready signal. + @return: The success status of sending the ready signal. + """ try: get_metadata_from_fabric(None, lease['unknown-245']) + return True except Exception as e: report_diagnostic_event( "Error communicating with Azure fabric; You may experience " "connectivity issues: %s" % e, logger_func=LOG.warning) + return False def _should_reprovision(self, ret): """Whether or not we should poll IMDS for reprovisioning data. diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3b9456f7..56c1cf18 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1161,6 +1161,19 @@ scbus-1 on xpt0 bus 0 dsrc = self._get_ds({'ovfcontent': xml}) dsrc.get_data() + def test_dsaz_report_ready_returns_true_when_report_succeeds( + self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + self.assertTrue(dsrc._report_ready(lease=mock.MagicMock())) + + def test_dsaz_report_ready_returns_false_and_does_not_propagate_exc( + self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + self.get_metadata_from_fabric.side_effect = Exception + self.assertFalse(dsrc._report_ready(lease=mock.MagicMock())) + def test_exception_fetching_fabric_data_doesnt_propagate(self): """Errors communicating with fabric should warn, but return True.""" dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) @@ -2041,7 +2054,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): @mock.patch('time.sleep', mock.MagicMock()) @mock.patch(MOCKPATH + 'EphemeralDHCPv4') def test_poll_imds_re_dhcp_on_timeout(self, m_dhcpv4, report_ready_func, - fake_resp, m_media_switch, m_dhcp, + m_request, m_media_switch, m_dhcp, m_net): """The poll_imds will retry DHCP on IMDS timeout.""" report_file = self.tmp_path('report_marker', self.tmp) @@ -2070,7 +2083,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): # Third try should succeed and stop retries or redhcp return mock.MagicMock(status_code=200, text="good", content="good") - fake_resp.side_effect = fake_timeout_once + m_request.side_effect = fake_timeout_once dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): @@ -2080,11 +2093,10 @@ class TestPreprovisioningPollIMDS(CiTestCase): self.assertEqual(3, m_dhcpv4.call_count, 'Expected 3 DHCP calls') self.assertEqual(4, self.tries, 'Expected 4 total reads from IMDS') - def test_poll_imds_report_ready_false(self, - report_ready_func, fake_resp, - m_media_switch, m_dhcp, m_net): - """The poll_imds should not call reporting ready - when flag is false""" + def test_does_not_poll_imds_report_ready_when_marker_file_exists( + self, m_report_ready, m_request, m_media_switch, m_dhcp, m_net): + """poll_imds should not call report ready when the reported ready + marker file exists""" report_file = self.tmp_path('report_marker', self.tmp) write_file(report_file, content='dont run report_ready :)') m_dhcp.return_value = [{ @@ -2095,11 +2107,49 @@ class TestPreprovisioningPollIMDS(CiTestCase): dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): dsa._poll_imds() - self.assertEqual(report_ready_func.call_count, 0) + self.assertEqual(m_report_ready.call_count, 0) + + def test_poll_imds_report_ready_success_writes_marker_file( + self, m_report_ready, m_request, m_media_switch, m_dhcp, m_net): + """poll_imds should write the report_ready marker file if + reporting ready succeeds""" + report_file = self.tmp_path('report_marker', self.tmp) + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'}] + m_media_switch.return_value = None + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + self.assertFalse(os.path.exists(report_file)) + with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): + dsa._poll_imds() + self.assertEqual(m_report_ready.call_count, 1) + self.assertTrue(os.path.exists(report_file)) + + def test_poll_imds_report_ready_failure_raises_exc_and_doesnt_write_marker( + self, m_report_ready, m_request, m_media_switch, m_dhcp, m_net): + """poll_imds should write the report_ready marker file if + reporting ready succeeds""" + report_file = self.tmp_path('report_marker', self.tmp) + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'}] + m_media_switch.return_value = None + m_report_ready.return_value = False + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + self.assertFalse(os.path.exists(report_file)) + with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): + self.assertRaises( + InvalidMetaDataException, + dsa._poll_imds) + self.assertEqual(m_report_ready.call_count, 1) + self.assertFalse(os.path.exists(report_file)) -@mock.patch(MOCKPATH + 'subp.subp') -@mock.patch(MOCKPATH + 'util.write_file') +@mock.patch(MOCKPATH + 'DataSourceAzure._report_ready', mock.MagicMock()) +@mock.patch(MOCKPATH + 'subp.subp', mock.MagicMock()) +@mock.patch(MOCKPATH + 'util.write_file', mock.MagicMock()) @mock.patch(MOCKPATH + 'util.is_FreeBSD') @mock.patch('cloudinit.sources.helpers.netlink.' 'wait_for_media_disconnect_connect') @@ -2115,10 +2165,10 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): self.paths = helpers.Paths({'cloud_dir': tmp}) dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d - def test_poll_imds_returns_ovf_env(self, fake_resp, + def test_poll_imds_returns_ovf_env(self, m_request, m_dhcp, m_net, m_media_switch, - m_is_bsd, write_f, subp): + m_is_bsd): """The _poll_imds method should return the ovf_env.xml.""" m_is_bsd.return_value = False m_media_switch.return_value = None @@ -2128,11 +2178,11 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02' host = "169.254.169.254" full_url = url.format(host) - fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf", + m_request.return_value = mock.MagicMock(status_code=200, text="ovf", content="ovf") dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) self.assertTrue(len(dsa._poll_imds()) > 0) - self.assertEqual(fake_resp.call_args_list, + self.assertEqual(m_request.call_args_list, [mock.call(allow_redirects=True, headers={'Metadata': 'true', 'User-Agent': @@ -2147,10 +2197,10 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): static_routes=None) self.assertEqual(m_net.call_count, 2) - def test__reprovision_calls__poll_imds(self, fake_resp, + def test__reprovision_calls__poll_imds(self, m_request, m_dhcp, m_net, m_media_switch, - m_is_bsd, write_f, subp): + m_is_bsd): """The _reprovision method should call poll IMDS.""" m_is_bsd.return_value = False m_media_switch.return_value = None @@ -2165,7 +2215,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): username = "myuser" odata = {'HostName': hostname, 'UserName': username} content = construct_valid_ovf_env(data=odata) - fake_resp.return_value = mock.MagicMock(status_code=200, text=content, + m_request.return_value = mock.MagicMock(status_code=200, text=content, content=content) dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) md, _ud, cfg, _d = dsa._reprovision() @@ -2182,7 +2232,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, url=full_url ), - fake_resp.call_args_list) + m_request.call_args_list) self.assertEqual(m_dhcp.call_count, 2) m_net.assert_any_call( broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5c31b8be..6e004e34 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -742,7 +742,8 @@ class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase): self.assertEqual(1, shim.return_value.clean_up.call_count) @mock.patch.object(azure_helper, 'WALinuxAgentShim') - def test_failure_in_registration_calls_clean_up(self, shim): + def test_failure_in_registration_propagates_exc_and_calls_clean_up( + self, shim): shim.return_value.register_with_azure_and_fetch_data.side_effect = ( SentinelException) self.assertRaises(SentinelException, -- cgit v1.2.3 From b94962b558e929a365bcfad1ca9a9445eee575e8 Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Mon, 19 Oct 2020 16:09:51 -0300 Subject: Add more integration tests (#615) Translate the following tests from `cloud_tests` to the new integration test framework: * test_runcmd.py * seed_random_data.py * set_hostname.py * set_hostname_fqdn.py * snap.py * ssh_auth_key_fingerprints_disable.py * ssh_auth_key_fingerprints_enable.py * ssh_import_id.py * ssh_keys_generate.py * ssh_keys_provided.py * timezone.py * write_files.py --- .../modules/test_package_update_upgrade_install.py | 74 ++++++++++++ tests/integration_tests/modules/test_runcmd.py | 24 ++++ .../modules/test_seed_random_data.py | 27 +++++ .../integration_tests/modules/test_set_hostname.py | 46 +++++++ tests/integration_tests/modules/test_snap.py | 28 +++++ .../modules/test_ssh_auth_key_fingerprints.py | 47 ++++++++ .../integration_tests/modules/test_ssh_generate.py | 50 ++++++++ .../modules/test_ssh_import_id.py | 28 +++++ .../modules/test_ssh_keys_provided.py | 134 +++++++++++++++++++++ tests/integration_tests/modules/test_timezone.py | 24 ++++ .../integration_tests/modules/test_write_files.py | 65 ++++++++++ 11 files changed, 547 insertions(+) create mode 100644 tests/integration_tests/modules/test_package_update_upgrade_install.py create mode 100644 tests/integration_tests/modules/test_runcmd.py create mode 100644 tests/integration_tests/modules/test_seed_random_data.py create mode 100644 tests/integration_tests/modules/test_set_hostname.py create mode 100644 tests/integration_tests/modules/test_snap.py create mode 100644 tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py create mode 100644 tests/integration_tests/modules/test_ssh_generate.py create mode 100644 tests/integration_tests/modules/test_ssh_import_id.py create mode 100644 tests/integration_tests/modules/test_ssh_keys_provided.py create mode 100644 tests/integration_tests/modules/test_timezone.py create mode 100644 tests/integration_tests/modules/test_write_files.py diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py new file mode 100644 index 00000000..8a38ad84 --- /dev/null +++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py @@ -0,0 +1,74 @@ +"""Integration test for the package update upgrade install module. + +This test module asserts that packages are upgraded/updated during boot +with the ``package_update_upgrade_install`` module. We are also testing +if we can install new packages during boot too. + +(This is ported from +``tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml``.) + +NOTE: the testcase for this looks for the command in history.log as + /usr/bin/apt-get..., which is not how it always appears. it should + instead look for just apt-get... +""" + +import re +import pytest + + +USER_DATA = """\ +#cloud-config +packages: + - sl + - tree +package_update: true +package_upgrade: true +""" + + +@pytest.mark.user_data(USER_DATA) +class TestPackageUpdateUpgradeInstall: + + def assert_package_installed(self, pkg_out, name, version=None): + """Check dpkg-query --show output for matching package name. + + @param name: package base name + @param version: string representing a package version or part of a + version. + """ + pkg_match = re.search( + "^%s\t(?P.*)$" % name, pkg_out, re.MULTILINE) + if pkg_match: + installed_version = pkg_match.group("version") + if not version: + return # Success + if installed_version.startswith(version): + return # Success + raise AssertionError( + "Expected package version %s-%s not found. Found %s" % + name, version, installed_version) + raise AssertionError("Package not installed: %s" % name) + + def test_new_packages_are_installed(self, class_client): + pkg_out = class_client.execute("dpkg-query --show") + + self.assert_package_installed(pkg_out, "sl") + self.assert_package_installed(pkg_out, "tree") + + def test_packages_were_updated(self, class_client): + out = class_client.execute( + "grep ^Commandline: /var/log/apt/history.log") + assert ( + "Commandline: /usr/bin/apt-get --option=Dpkg::Options" + "::=--force-confold --option=Dpkg::options::=--force-unsafe-io " + "--assume-yes --quiet install sl tree") in out + + def test_packages_were_upgraded(self, class_client): + """Test cloud-init-output for install & upgrade stuff.""" + out = class_client.read_from_file("/var/log/cloud-init-output.log") + assert "Setting up tree (" in out + assert "Setting up sl (" in out + assert "Reading package lists..." in out + assert "Building dependency tree..." in out + assert "Reading state information..." in out + assert "Calculating upgrade..." in out diff --git a/tests/integration_tests/modules/test_runcmd.py b/tests/integration_tests/modules/test_runcmd.py new file mode 100644 index 00000000..eabe778d --- /dev/null +++ b/tests/integration_tests/modules/test_runcmd.py @@ -0,0 +1,24 @@ +"""Integration test for the runcmd module. + +This test specifies a command to be executed by the ``runcmd`` module +and then checks if that command was executed during boot. + +(This is ported from +``tests/cloud_tests/testcases/modules/runcmd.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +runcmd: + - echo cloud-init run cmd test > /var/tmp/run_cmd +""" + + +class TestRuncmd: + + @pytest.mark.user_data(USER_DATA) + def test_runcmd(self, client): + runcmd_output = client.read_from_file("/var/tmp/run_cmd") + assert runcmd_output.strip() == "cloud-init run cmd test" diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py new file mode 100644 index 00000000..db3d2193 --- /dev/null +++ b/tests/integration_tests/modules/test_seed_random_data.py @@ -0,0 +1,27 @@ +"""Integration test for the random seed module. + +This test specifies a command to be executed by the ``seed_random`` module, by +providing a different data to be used as seed data. We will then check +if that seed data was actually used. + +(This is ported from +``tests/cloud_tests/testcases/modules/seed_random_data.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed +""" + + +class TestSeedRandomData: + + @pytest.mark.user_data(USER_DATA) + def test_seed_random_data(self, client): + seed_output = client.read_from_file("/root/seed") + assert seed_output.strip() == "MYUb34023nD:LFDK10913jk;dfnk:Df" diff --git a/tests/integration_tests/modules/test_set_hostname.py b/tests/integration_tests/modules/test_set_hostname.py new file mode 100644 index 00000000..ff46feb9 --- /dev/null +++ b/tests/integration_tests/modules/test_set_hostname.py @@ -0,0 +1,46 @@ +"""Integration test for the set_hostname module. + +This module specify two tests: One updates only the hostname and the other +one updates the hostname and fqdn of the system. For both of these tests +we will check is the changes requested by the user data are being respected +after the system is boot. + +(This is ported from +``tests/cloud_tests/testcases/modules/set_hostname.yaml`` and +``tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml``.)""" + +import pytest + + +USER_DATA_HOSTNAME = """\ +#cloud-config +hostname: cloudinit2 +""" + +USER_DATA_FQDN = """\ +#cloud-config +manage_etc_hosts: true +hostname: cloudinit1 +fqdn: cloudinit2.i9n.cloud-init.io +""" + + +class TestHostname: + + @pytest.mark.user_data(USER_DATA_HOSTNAME) + def test_hostname(self, client): + hostname_output = client.execute("hostname") + assert "cloudinit2" in hostname_output.strip() + + @pytest.mark.user_data(USER_DATA_FQDN) + def test_hostname_and_fqdn(self, client): + hostname_output = client.execute("hostname") + assert "cloudinit1" in hostname_output.strip() + + fqdn_output = client.execute("hostname --fqdn") + assert "cloudinit2.i9n.cloud-init.io" in fqdn_output.strip() + + host_output = client.execute("grep ^127 /etc/hosts") + assert '127.0.1.1 {} {}'.format( + fqdn_output, hostname_output) in host_output + assert '127.0.0.1 localhost' in host_output diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py new file mode 100644 index 00000000..d78a0b1e --- /dev/null +++ b/tests/integration_tests/modules/test_snap.py @@ -0,0 +1,28 @@ +"""Integration test for the snap module. + +This test specifies a command to be executed by the ``snap`` module +and then checks that if that command was executed during boot. + +(This is ported from +``tests/cloud_tests/testcases/modules/runcmd.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +package_update: true +snap: + squashfuse_in_container: true + commands: + - snap install hello-world +""" + + +class TestSnap: + + @pytest.mark.user_data(USER_DATA) + def test_snap(self, client): + snap_output = client.execute("snap list") + assert "core " in snap_output + assert "hello-world " in snap_output diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py new file mode 100644 index 00000000..e88d9a02 --- /dev/null +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -0,0 +1,47 @@ +"""Integration test for the ssh_authkey_fingerprints module. + +This modules specifies two tests regarding the ``ssh_authkey_fingerprints`` +module. The first one verifies that we can disable the module behavior while +the second one verifies if the module is working as expected if enabled. + +(This is ported from +``tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml``, +``tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.yaml``. +)""" +import re + +import pytest + + +USER_DATA_SSH_AUTHKEY_DISABLE = """\ +#cloud-config +no_ssh_fingerprints: true +""" + +USER_DATA_SSH_AUTHKEY_ENABLE="""\ +#cloud-config +ssh_genkeytypes: + - ecdsa + - ed25519 +ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== +""" # noqa + + +class TestSshAuthkeyFingerprints: + + @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_DISABLE) + def test_ssh_authkey_fingerprints_disable(self, client): + cloudinit_output = client.read_from_file("/var/log/cloud-init.log") + assert ( + "Skipping module named ssh-authkey-fingerprints, " + "logging of SSH fingerprints disabled") in cloudinit_output + + @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_ENABLE) + def test_ssh_authkey_fingerprints_enable(self, client): + syslog_output = client.read_from_file("/var/log/syslog") + + assert re.search(r'256 SHA256:.*(ECDSA)', syslog_output) is not None + assert re.search(r'256 SHA256:.*(ED25519)', syslog_output) is not None + assert re.search(r'1024 SHA256:.*(DSA)', syslog_output) is None + assert re.search(r'2048 SHA256:.*(RSA)', syslog_output) is None diff --git a/tests/integration_tests/modules/test_ssh_generate.py b/tests/integration_tests/modules/test_ssh_generate.py new file mode 100644 index 00000000..8c60fb87 --- /dev/null +++ b/tests/integration_tests/modules/test_ssh_generate.py @@ -0,0 +1,50 @@ +"""Integration test for the ssh module. + +This module has two tests to verify if we can create ssh keys +through the ``ssh`` module. The first test asserts that some keys +were not created while the second one verifies if the expected +keys were created. + +(This is ported from +``tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +ssh_genkeytypes: + - ecdsa + - ed25519 +authkey_hash: sha512 +""" + + +@pytest.mark.user_data(USER_DATA) +class TestSshKeysGenerate: + + @pytest.mark.parametrize( + "ssh_key_path", ( + "/etc/ssh/ssh_host_dsa_key.pub", + "/etc/ssh/ssh_host_dsa_key", + "/etc/ssh/ssh_host_rsa_key.pub", + "/etc/ssh/ssh_host_rsa_key", + ) + ) + def test_ssh_keys_not_generated(self, ssh_key_path, class_client): + out = class_client.execute( + "test -e {}".format(ssh_key_path) + ) + assert out.failed + + @pytest.mark.parametrize( + "ssh_key_path", ( + "/etc/ssh/ssh_host_ecdsa_key.pub", + "/etc/ssh/ssh_host_ecdsa_key", + "/etc/ssh/ssh_host_ed25519_key.pub", + "/etc/ssh/ssh_host_ed25519_key", + ) + ) + def test_ssh_keys_generated(self, ssh_key_path, class_client): + out = class_client.read_from_file(ssh_key_path) + assert "" != out.strip() diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py new file mode 100644 index 00000000..2f2ac92c --- /dev/null +++ b/tests/integration_tests/modules/test_ssh_import_id.py @@ -0,0 +1,28 @@ +"""Integration test for the ssh_import_id module. + +This test specifies ssh keys to be imported by the ``ssh_import_id`` module +and then checks that if the ssh keys were successfully imported. + +(This is ported from +``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +ssh_import_id: + - gh:powersj + - lp:smoser +""" + + +class TestSshImportId: + + @pytest.mark.user_data(USER_DATA) + def test_ssh_import_id(self, client): + ssh_output = client.read_from_file( + "/home/ubuntu/.ssh/authorized_keys") + + assert '# ssh-import-id gh:powersj' in ssh_output + assert '# ssh-import-id lp:smoser' in ssh_output diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py new file mode 100644 index 00000000..4699518d --- /dev/null +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -0,0 +1,134 @@ +"""Integration test for the ssh module. + +This test specifies keys to be provided to the system through the ``ssh`` +module and then checks that if those keys were successfully added to the +system. + +(This is ported from +``tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml''``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +disable_root: false +ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== +ssh_keys: + rsa_private: | + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEAtPx6PqN3iSEsnTtibyIEy52Tra8T5fn0ryXyg46Di2NBwdnj + o8trNv9jenfV/UhmePl58lXjT43wV8OCMl6KsYXyBdegM35NNtono4I4mLLKFMR9 + 9TOtDn6iYcaNenVhF3ZCj9Z2nNOlTrdc0uchHqKMrxLjCRCUrL91Uf+xioTF901Y + RM+ZqC5lT92yAL76F4qPF+Lq1QtUfNfUIwwvOp5ccDZLPxij0YvyBzubYye9hJHu + yjbJv78R4JHV+L2WhzSoX3W/6WrxVzeXqFGqH894ccOaC/7tnqSP6V8lIQ6fE2+c + DurJcpM3CJRgkndGHjtU55Y71YkcdLksSMvezQIDAQABAoIBAQCrU4IJP8dNeaj5 + IpkY6NQvR/jfZqfogYi+MKb1IHin/4rlDfUvPcY9pt8ttLlObjYK+OcWn3Vx/sRw + 4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2unRQvLZpMRdywBm + lq95OrCghnG03aUsFJUZPpi5ydnwbA12ma+KHkG0EzaVlhA7X9N6z0K6U+zue2gl + goMLt/MH0rsYawkHrwiwXaIFQeyV4MJP0vmrZLbFk1bycu9X/xPtTYotWyWo4eKA + cb05uu04qwexkKHDM0KXtT0JecbTo2rOefFo8Uuab6uJY+fEHNocZ+v1vLA4aOxJ + ovp1JuXlAoGBAOWYNgKrlTfy5n0sKsNk+1RuL2jHJZJ3HMd0EIt7/fFQN3Fi08Hu + jtntqD30Wj+DJK8b8Lrt66FruxyEJm5VhVmwkukrLR5ige2f6ftZnoFCmdyy+0zP + dnPZSUe2H5ZPHa+qthJgHLn+al2P04tGh+1fGHC2PbP+e0Co+/ZRIOxrAoGBAMnN + IEen9/FRsqvnDd36I8XnJGskVRTZNjylxBmbKcuMWm+gNhOI7gsCAcqzD4BYZjjW + pLhrt/u9p+l4MOJy6OUUdM/okg12SnJEGryysOcVBcXyrvOfklWnANG4EAH5jt1N + ftTb1XTxzvWVuR/WJK0B5MZNYM71cumBdUDtPi+nAoGAYmoIXMSnxb+8xNL10aOr + h9ljQQp8NHgSQfyiSufvRk0YNuYh1vMnEIsqnsPrG2Zfhx/25GmvoxXGssaCorDN + 5FAn6QK06F1ZTD5L0Y3sv4OI6G1gAuC66ZWuL6sFhyyKkQ4f1WiVZ7SCa3CHQSAO + i9VDaKz1bf4bXvAQcNj9v9kCgYACSOZCqW4vN0OUmqsXhkt9ZB6Pb/veno70pNPR + jmYsvcwQU3oJQpWfXkhy6RAV3epaXmPDCsUsfns2M3wqNC7a2R5xdCqjKGGzZX4A + AO3rz9se4J6Gd5oKijeCKFlWDGNHsibrdgm2pz42nZlY+O21X74dWKbt8O16I1MW + hxkbJQKBgAXfuen/srVkJgPuqywUYag90VWCpHsuxdn+fZJa50SyZADr+RbiDfH2 + vek8Uo8ap8AEsv4Rfs9opUcUZevLp3g2741eOaidHVLm0l4iLIVl03otGOqvSzs+ + A3tFPEOxauXpzCt8f8eXsz0WQXAgIKW2h8zu5QHjomioU3i27mtE + -----END RSA PRIVATE KEY----- + rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97N root@xenial-lxd + dsa_private: | + -----BEGIN DSA PRIVATE KEY----- + MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXrhOVAfzZ6+jklP + 55mzvC7jO53PWWC31hq10xBoWdev0WtcNF9Tv+4bAa1263y51Rqo4GI7xx+xic1d + mLqqfYijBT9k48J/1tV0cs1Wjs6FP/IJTD/kYVC930JjYQMi722lBnUxsQIVAL7i + z3fTGKTvSzvW0wQlwnYpS2QFAoGANp+KdyS9V93HgxGQEN1rlj/TSv/a3EVdCKtE + nQf55aPHxDAVDVw5JtRh4pZbbRV4oGRPc9KOdjo5BU28vSM3Lmhkb+UaaDXwHkgI + nK193o74DKjADWZxuLyyiKHiMOhxozoxDfjWxs8nz6uqvSW0pr521EwIY6RajbED + nZ2a3GkCgYEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pf + Q2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2E + wExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkICFA5kVUcW + nCPOXEQsayANi8+Cb7BH + -----END DSA PRIVATE KEY----- + dsa_public: ssh-dss AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4RZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM7nc9ZYLfWGrXTEGhZ16/Ra1w0X1O/7hsBrXbrfLnVGqjgYjvHH7GJzV2Yuqp9iKMFP2Tjwn/W1XRyzVaOzoU/8glMP+RhUL3fQmNhAyLvbaUGdTGxAAAAFQC+4s930xik70s71tMEJcJ2KUtkBQAAAIA2n4p3JL1X3ceDEZAQ3WuWP9NK/9rcRV0Iq0SdB/nlo8fEMBUNXDkm1GHillttFXigZE9z0o52OjkFTby9IzcuaGRv5RpoNfAeSAicrX3ejvgMqMANZnG4vLKIoeIw6HGjOjEN+NbGzyfPq6q9JbSmvnbUTAhjpFqNsQOdnZrcaQAAAIEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pfQ2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2EwExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkI= root@xenial-lxd + ed25519_private: | + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+QAAAJgwt+lcMLfp + XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+Q + AAAEDQlFZpz9q8+/YJHS9+jPAqy2ZT6cGEv8HTB6RZtTjd/dudAZSu4vjZpVWzId5pXmZg + 1M6G15dqjQ2XkNVOEnb5AAAAD3Jvb3RAeGVuaWFsLWx4ZAECAwQFBg== + -----END OPENSSH PRIVATE KEY----- + ed25519_public: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6G15dqjQ2XkNVOEnb5 root@xenial-lxd + ecdsa_private: | + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIDuK+QFc1wmyJY8uDqQVa1qHte30Rk/fdLxGIBkwJAyOoAoGCCqGSM49 + AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY5mpZqxgX4vcgb + 7f/CtXuM6s2svcDJqAeXr6Wk8OJJcMxylA== + -----END EC PRIVATE KEY----- + ecdsa_public: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFsS5Tvky/IC/dXhE/afxxUG6kdQOvdQJCYGZN42OZqWasYF+L3IG+3/wrV7jOrNrL3AyagHl6+lpPDiSXDMcpQ= root@xenial-lxd +""" # noqa + + +@pytest.mark.user_data(USER_DATA) +class TestSshKeysProvided: + + def test_ssh_dsa_keys_provided(self, class_client): + """Test dsa public key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_dsa_key.pub") + assert ( + "AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4R" + "ZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM") in out + + """Test dsa private key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_dsa_key") + assert ( + "MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXr" + "hOVAfzZ6+jklP") in out + + def test_ssh_rsa_keys_provided(self, class_client): + """Test rsa public key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key.pub") + assert ( + "AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgT" + "LnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4") in out + + """Test rsa private key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key") + assert ( + "4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un" + "RQvLZpMRdywBm") in out + + def test_ssh_ecdsa_keys_provided(self, class_client): + """Test ecdsa public key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key.pub") + assert ( + "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAAB" + "BBFsS5Tvky/IC/dXhE/afxxU") in out + + """Test ecdsa private key generated.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key") + assert ( + "AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY" + "5mpZqxgX4vcgb") in out + + def test_ssh_ed25519_keys_provided(self, class_client): + """Test ed25519 public key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_ed25519_key.pub") + assert ( + "AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6" + "G15dqjQ2XkNVOEnb5") in out + + """Test ed25519 private key was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_ed25519_key") + assert ( + "XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNT" + "OhteXao0Nl5DVThJ2+Q") in out diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py new file mode 100644 index 00000000..6080d79e --- /dev/null +++ b/tests/integration_tests/modules/test_timezone.py @@ -0,0 +1,24 @@ +"""Integration test for the timezone module. + +This test specifies a timezone to be used by the ``timezone`` module +and then checks that if that timezone was respected during boot. + +(This is ported from +``tests/cloud_tests/testcases/modules/timezone.yaml``.)""" + +import pytest + + +USER_DATA = """\ +#cloud-config +timezone: US/Aleutian +""" + + +class TestTimezone: + + @pytest.mark.user_data(USER_DATA) + def test_timezone(self, client): + timezone_output = client.execute( + 'date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400"') + assert timezone_output.strip() == "HDT" diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py new file mode 100644 index 00000000..d7032a0c --- /dev/null +++ b/tests/integration_tests/modules/test_write_files.py @@ -0,0 +1,65 @@ +"""Integration test for the write_files module. + +This test specifies files to be created by the ``write_files`` module +and then checks if those files were created during boot. + +(This is ported from +``tests/cloud_tests/testcases/modules/write_files.yaml``.)""" + +import base64 +import pytest + + +ASCII_TEXT = "ASCII text" +B64_CONTENT = base64.b64encode(ASCII_TEXT.encode("utf-8")) + +# NOTE: the binary data can be any binary data, not only executables +# and can be generated via the base 64 command as such: +# $ base64 < hello > hello.txt +# the opposite is running: +# $ base64 -d < hello.txt > hello +# +USER_DATA = """\ +#cloud-config +write_files: +- encoding: b64 + content: {} + owner: root:root + path: /root/file_b64 + permissions: '0644' +- content: | + # My new /root/file_text + + SMBDOPTIONS="-D" + path: /root/file_text +- content: !!binary | + /Z/xrHR4WINT0UNoKPQKbuovp6+Js+JK + path: /root/file_binary + permissions: '0555' +- encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /root/file_gzip + permissions: '0755' +""".format(B64_CONTENT.decode("ascii")) + + +@pytest.mark.user_data(USER_DATA) +class TestWriteFiles: + + @pytest.mark.parametrize( + "cmd,expected_out", ( + ("file /root/file_b64", ASCII_TEXT), + ("md5sum Date: Mon, 19 Oct 2020 22:59:16 +0200 Subject: bddeb: new --packaging-branch argument to pull packaging from branch (#576) bddeb builds a .deb package using the template packaging files in packages/debian/. The new --packaging-branch flag allows to specify a git branch where to pull the packaging (i.e. the debian/ directory) from. This is useful to build a .deb package from master with the very same packaging which is used for the uploads. --- cloudinit/subp.py | 6 ++-- doc/rtd/topics/cloud_tests.rst | 13 ++++--- packages/bddeb | 80 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 3e4efa42..024e1a98 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -144,7 +144,7 @@ class ProcessExecutionError(IOError): def subp(args, data=None, rcs=None, env=None, capture=True, combine_capture=False, shell=False, logstring=False, decode="replace", target=None, update_env=None, - status_cb=None): + status_cb=None, cwd=None): """Run a subprocess. :param args: command to run in a list. [cmd, arg1, arg2...] @@ -181,6 +181,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, :param status_cb: call this fuction with a single string argument before starting and after finishing. + :param cwd: + change the working directory to cwd before executing the command. :return if not capturing, return is (None, None) @@ -254,7 +256,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, try: sp = subprocess.Popen(bytes_args, stdout=stdout, stderr=stderr, stdin=stdin, - env=env, shell=shell) + env=env, shell=shell, cwd=cwd) (out, err) = sp.communicate(data) except OSError as e: if status_cb: diff --git a/doc/rtd/topics/cloud_tests.rst b/doc/rtd/topics/cloud_tests.rst index e4e893d2..0fbb1301 100644 --- a/doc/rtd/topics/cloud_tests.rst +++ b/doc/rtd/topics/cloud_tests.rst @@ -151,17 +151,20 @@ cloud-init located in a different directory, use the option ``--cloud-init Bddeb ----- -The ``bddeb`` command can be used to generate a deb file. This is used by -the tree_run and tree_collect commands to build a deb of the current -working tree. It can also be used a user to generate a deb for use in other -situations and avoid needing to have all the build and test dependencies -installed locally. +The ``bddeb`` command can be used to generate a deb file. This is used by the +tree_run and tree_collect commands to build a deb of the current working tree +using the packaging template contained in the ``packages/debian/`` directory. +It can also be used to generate a deb for use in other situations and avoid +needing to have all the build and test dependencies installed locally. * ``--bddeb-args``: arguments to pass through to bddeb * ``--build-os``: distribution to use as build system (default is xenial) * ``--build-platform``: platform to use for build system (default is lxd) * ``--cloud-init``: path to base of cloud-init tree (default is '.') * ``--deb``: path to write output deb to (default is '.') +* ``--packaging-branch``: import the ``debian/`` packaging directory + from the specified branch (default: ``ubuntu/devel``) instead of using + the packaging template. Setup Image ----------- diff --git a/packages/bddeb b/packages/bddeb index b0f219b6..a3fb8848 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -5,6 +5,7 @@ import csv import json import os import shutil +import subprocess import sys UNRELEASED = "UNRELEASED" @@ -99,6 +100,36 @@ def write_debian_folder(root, templ_data, cloud_util_deps): params={'build_depends': ','.join(requires)}) +def write_debian_folder_from_branch(root, templ_data, branch): + """Import a debian package directory from a branch.""" + print("Importing debian/ from branch %s to %s" % (branch, root)) + + p_dumpdeb = subprocess.Popen( + ["git", "archive", branch, "debian"], stdout=subprocess.PIPE + ) + subprocess.check_call( + ["tar", "-v", "-C", root, "-x"], + stdin=p_dumpdeb.stdout + ) + + print("Adding new entry to debian/changelog") + full_deb_version = ( + templ_data["version_long"] + "-1~bddeb" + templ_data["release_suffix"] + ) + subp.subp( + [ + "dch", + "--distribution", + templ_data["debian_release"], + "--newversion", + full_deb_version, + "--controlmaint", + "Snapshot build.", + ], + cwd=root + ) + + def read_version(): return json.loads(run_helper('read-version', ['--json'])) @@ -140,6 +171,15 @@ def get_parser(): parser.add_argument("--signuser", default=False, action='store', help="user to sign, see man dpkg-genchanges") + + parser.add_argument("--packaging-branch", nargs="?", metavar="BRANCH", + const="ubuntu/devel", type=str, + help=( + "Import packaging from %(metavar)s instead of" + " using the packages/debian/* templates" + " (default: %(const)s)" + )) + return parser @@ -147,6 +187,37 @@ def main(): parser = get_parser() args = parser.parse_args() + if args.packaging_branch: + try: + subp.subp( + [ + "git", + "show-ref", + "--quiet", + "--verify", + "refs/heads/" + args.packaging_branch, + ] + ) + except subp.ProcessExecutionError: + print("Couldn't find branch '%s'." % args.packaging_branch) + print("You may need to checkout the branch from the git remote.") + return 1 + try: + subp.subp( + [ + "git", + "cat-file", + "-e", + args.packaging_branch + ":debian/control", + ] + ) + except subp.ProcessExecutionError: + print( + "Couldn't find debian/control in branch '%s'." + " Is it a packaging branch?" % args.packaging_branch + ) + return 1 + if not args.sign: args.debuild_args.extend(['-us', '-uc']) @@ -198,7 +269,14 @@ def main(): xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long']) templ_data.update(ver_data) - write_debian_folder(xdir, templ_data, cloud_util_deps=args.cloud_utils) + if args.packaging_branch: + write_debian_folder_from_branch( + xdir, templ_data, args.packaging_branch + ) + else: + write_debian_folder( + xdir, templ_data, cloud_util_deps=args.cloud_utils + ) print("Running 'debuild %s' in %r" % (' '.join(args.debuild_args), xdir)) -- cgit v1.2.3 From b0e73814db4027dba0b7dc0282e295b7f653325c Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Tue, 20 Oct 2020 18:04:59 +0200 Subject: ssh_util: handle non-default AuthorizedKeysFile config (#586) The following commit merged all ssh keys into a default user file `~/.ssh/authorized_keys` in sshd_config had multiple files configured for AuthorizedKeysFile: commit f1094b1a539044c0193165a41501480de0f8df14 Author: Eduardo Otubo Date: Thu Dec 5 17:37:35 2019 +0100 Multiple file fix for AuthorizedKeysFile config (#60) This commit ignored the case when sshd_config would have a single file for AuthorizedKeysFile, but a non default configuration, for example `~/.ssh/authorized_keys_foobar`. In this case cloud-init would grab all keys from this file and write a new one, the default `~/.ssh/authorized_keys` causing the bug. rhbz: #1862967 Signed-off-by: Eduardo Otubo --- cloudinit/ssh_util.py | 6 +++--- tests/unittests/test_sshutil.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index c08042d6..d5113996 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -262,13 +262,13 @@ def extract_authorized_keys(username, sshd_cfg_file=DEF_SSHD_CFG): except (IOError, OSError): # Give up and use a default key filename - auth_key_fns[0] = default_authorizedkeys_file + auth_key_fns.append(default_authorizedkeys_file) util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in SSH " "config from %r, using 'AuthorizedKeysFile' file " "%r instead", DEF_SSHD_CFG, auth_key_fns[0]) - # always store all the keys in the user's private file - return (default_authorizedkeys_file, parse_authorized_keys(auth_key_fns)) + # always store all the keys in the first file configured on sshd_config + return (auth_key_fns[0], parse_authorized_keys(auth_key_fns)) def setup_user_keys(keys, username, options=None): diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index fd1d1bac..88a111e3 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -593,7 +593,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): fpw.pw_name, sshd_config) content = ssh_util.update_authorized_keys(auth_key_entries, []) - self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) + self.assertEqual(authorized_keys, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) self.assertTrue(VALID_CONTENT['dsa'] in content) @@ -610,7 +610,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( sshd_config, - "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) + "AuthorizedKeysFile %s %s" % (user_keys, authorized_keys) ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( @@ -618,7 +618,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): ) content = ssh_util.update_authorized_keys(auth_key_entries, []) - self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) + self.assertEqual(user_keys, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) self.assertTrue(VALID_CONTENT['dsa'] in content) -- cgit v1.2.3 From 468aede1384a5ba25c11a27db4e3e4b3f7b8e8b2 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 20 Oct 2020 15:50:47 -0400 Subject: .travis.yml: add integration-tests to Travis matrix (#600) This is implemented as a copy/paste of the `citest` integration testing script: the two are not intended to co-exist long-term, so it isn't worth further complicating an already complex part of our Travis configuration for short-term code reuse. The two changes from the `citest` definition: the test framework executed on the last line of `script`, and it is given a `name` so we can easily ignore failures. --- .travis.yml | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4c5bf4c4..87841424 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,8 @@ script: matrix: fast_finish: true + allow_failures: + - name: "Integration Tests (WIP)" include: - python: 3.6 env: @@ -119,6 +121,85 @@ matrix: - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' # Ubuntu LTS: Integration - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' + - name: "Integration Tests (WIP)" + if: NOT branch =~ /^ubuntu\// + cache: + - directories: + - lxd_images + - chroots + before_cache: + - | + # Find the most recent image file + latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)" + # This might be .rootfs or , normalise + latest_file="$(basename $latest_file .rootfs)" + # Find all files with that prefix and copy them to our cache dir + sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \; + install: + - git fetch --unshallow + - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper + - pip install . + - pip install tox + # bionic has lxd from deb installed, remove it first to ensure + # pylxd talks only to the lxd from snap + - sudo apt remove --purge lxd lxd-client + - sudo rm -Rf /var/lib/lxd + - sudo snap install lxd + - sudo lxd init --auto + - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles + # Move any cached lxd images into lxd's image dir + - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \; + - sudo usermod -a -G lxd $USER + - sudo sbuild-adduser $USER + - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc + script: + # Ubuntu LTS: Build + - ./packages/bddeb -S -d --release xenial + - | + needs_caching=false + if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then + # If we have a cached chroot, move it into place + sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 + sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 + # Write its configuration + cat > sbuild-xenial-amd64 << EOM + [xenial-amd64] + description=xenial-amd64 + groups=sbuild,root,admin + root-groups=sbuild,root,admin + # Uncomment these lines to allow members of these groups to access + # the -source chroots directly (useful for automated updates, etc). + #source-root-users=sbuild,root,admin + #source-root-groups=sbuild,root,admin + type=directory + profile=sbuild + union-type=overlay + directory=/var/lib/schroot/chroots/xenial-amd64 + EOM + sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/ + sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64 + # And ensure it's up-to-date. + before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)" + sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade" + after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum) + if [ "$before_pkgs" != "$after_pkgs" ]; then + needs_caching=true + fi + else + # Otherwise, create the chroot + sudo -E su $USER -c 'mk-sbuild xenial' + needs_caching=true + fi + # If there are changes to the schroot (or it's entirely new), + # tar up the schroot (to preserve ownership/permissions) and + # move it into the cached dir; no need to compress it because + # Travis will do that anyway + if [ "$needs_caching" = "true" ]; then + sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . + fi + # Use sudo to get a new shell where we're in the sbuild group + - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' + - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' - python: 3.5 env: TOXENV=xenial -- cgit v1.2.3 From 5f8a2bbc5f26c7abafbc9bd3d1b1b655ffdcc1ae Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 20 Oct 2020 17:13:37 -0400 Subject: cc_mounts: correctly fallback to dd if fallocate fails (#585) `create_swap()` was previously catching and not re-raising the ProcessExecutionError that indicated swap creation failure; this meant that the fallback logic could never be triggered. This commit adds the required re-raise (as well as removing a duplicated log message). LP: #1897099 --- cloudinit/config/cc_mounts.py | 8 ++++---- cloudinit/config/tests/test_mounts.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 54f2f878..c22d1698 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -255,8 +255,9 @@ def create_swapfile(fname: str, size: str) -> None: try: subp.subp(cmd, capture=True) except subp.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, method, e) + LOG.info(errmsg, fname, size, method, e) util.del_file(fname) + raise swap_dir = os.path.dirname(fname) util.ensure_dir(swap_dir) @@ -269,9 +270,8 @@ def create_swapfile(fname: str, size: str) -> None: else: try: create_swap(fname, size, "fallocate") - except subp.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, "dd", e) - LOG.warning("Will attempt with dd.") + except subp.ProcessExecutionError: + LOG.info("fallocate swap creation failed, will attempt with dd") create_swap(fname, size, "dd") if os.path.exists(fname): diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index 764a33e3..56510fd6 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -4,6 +4,7 @@ from unittest import mock import pytest from cloudinit.config.cc_mounts import create_swapfile +from cloudinit.subp import ProcessExecutionError M_PATH = 'cloudinit.config.cc_mounts.' @@ -26,3 +27,35 @@ class TestCreateSwapfile: create_swapfile(fname, '') assert mock.call(['mkswap', fname]) in m_subp.call_args_list + + @mock.patch(M_PATH + "util.get_mount_info") + @mock.patch(M_PATH + "subp.subp") + def test_fallback_from_fallocate_to_dd( + self, m_subp, m_get_mount_info, caplog, tmpdir + ): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + def subp_side_effect(cmd, *args, **kwargs): + # Mock fallocate failing, to initiate fallback + if cmd[0] == "fallocate": + raise ProcessExecutionError() + + m_subp.side_effect = subp_side_effect + # Use ext4 so both fallocate and dd are valid swap creation methods + m_get_mount_info.return_value = (mock.ANY, "ext4") + + create_swapfile(fname, "") + + cmds = [args[0][0] for args, _kwargs in m_subp.call_args_list] + assert "fallocate" in cmds, "fallocate was not called" + assert "dd" in cmds, "fallocate failure did not fallback to dd" + + assert cmds.index("dd") > cmds.index( + "fallocate" + ), "dd ran before fallocate" + + assert mock.call(["mkswap", fname]) in m_subp.call_args_list + + msg = "fallocate swap creation failed, will attempt with dd" + assert msg in caplog.text -- cgit v1.2.3 From 93cebe009d116230850c770227e9ead5c490c0d0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 21 Oct 2020 12:11:06 -0400 Subject: Drop vestigial update_resolve_conf_file function (#620) update_resolve_conf_file is no longer used. The last reference to it was removed in c3680475f9c970, which was itself a "remove dead code" commit. --- cloudinit/distros/rhel_util.py | 26 -------------------------- tests/unittests/test_distros/test_resolv.py | 6 ------ 2 files changed, 32 deletions(-) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 387a851f..d71394b4 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -8,7 +8,6 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.distros.parsers.resolv_conf import ResolvConf from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import log as logging @@ -50,29 +49,4 @@ def read_sysconfig_file(fn): contents = [] return (exists, SysConf(contents)) - -# Helper function to update RHEL/SUSE /etc/resolv.conf -def update_resolve_conf_file(fn, dns_servers, search_servers): - try: - r_conf = ResolvConf(util.load_file(fn)) - r_conf.parse() - except IOError: - util.logexc(LOG, "Failed at parsing %s reverting to an empty " - "instance", fn) - r_conf = ResolvConf('') - r_conf.parse() - if dns_servers: - for s in dns_servers: - try: - r_conf.add_nameserver(s) - except ValueError: - util.logexc(LOG, "Failed at adding nameserver %s", s) - if search_servers: - for s in search_servers: - try: - r_conf.add_search_domain(s) - except ValueError: - util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) - # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 68ea0083..7d940750 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -1,12 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.distros.parsers import resolv_conf -from cloudinit.distros import rhel_util from cloudinit.tests.helpers import TestCase import re -import tempfile BASE_RESOLVE = ''' @@ -24,10 +22,6 @@ class TestResolvHelper(TestCase): rp_r = str(rp).strip() self.assertEqual(BASE_RESOLVE, rp_r) - def test_write_works(self): - with tempfile.NamedTemporaryFile() as fh: - rhel_util.update_resolve_conf_file(fh.name, [], []) - def test_local_domain(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) self.assertIsNone(rp.local_domain) -- cgit v1.2.3 From 72d85ff98f4185db10af980776b1ba46fa340920 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 22 Oct 2020 11:26:08 -0700 Subject: docs: Add how to use cloud-localds to boot qemu (#617) * docs: Add hot to use cloud-localds to boot qemu There is a complete lack of documentation on using cloud-localds with cloud-init to boot an image locally. Drive by, added some more whitepapers, blogs, and videos * fix line length * * add where cloud-localds comes from * add more specific example with metadata and network config * Add link to cloud-utils package --- doc/rtd/topics/faq.rst | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index aa1be142..944cc27f 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -226,12 +226,65 @@ custom network config. .. _Instance Configuration: https://linuxcontainers.org/lxd/docs/master/instances .. _Custom Network Configuration: https://linuxcontainers.org/lxd/docs/master/cloud-init +cloud-localds +------------- + +The `cloud-localds` command from the `cloud-utils`_ package generates a disk +with user supplied data. The NoCloud datasouce allows users to provide their +own user data, metadata, or network configuration directly to an instance +without running a network service. This is helpful for launching local cloud +images with QEMU for example. + +The following is an example of creating the local disk using the cloud-localds +command: + +.. code-block:: shell-session + + $ cat >user-data < Date: Fri, 23 Oct 2020 15:20:18 -0400 Subject: stages: don't reset permissions of cloud-init.log every boot (#624) ensure_file needed modification to support doing this, so this commit also includes the following changes: test_util: add tests for util.ensure_file util: add preserve_mode parameter to ensure_file util: add (partial) type annotations to ensure_file LP: #1900837 --- cloudinit/stages.py | 2 +- cloudinit/tests/test_stages.py | 62 ++++++++++++++++++++++++++++++++++++++++++ cloudinit/tests/test_util.py | 45 ++++++++++++++++++++++++++++++ cloudinit/util.py | 8 ++++-- 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 765f4aab..0cce6e80 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -148,7 +148,7 @@ class Init(object): util.ensure_dirs(self._initial_subdirs()) log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') if log_file: - util.ensure_file(log_file) + util.ensure_file(log_file, preserve_mode=True) perms = self.cfg.get('syslog_fix_perms') if not perms: perms = {} diff --git a/cloudinit/tests/test_stages.py b/cloudinit/tests/test_stages.py index d5c9c0e4..d2d1b37f 100644 --- a/cloudinit/tests/test_stages.py +++ b/cloudinit/tests/test_stages.py @@ -3,6 +3,9 @@ """Tests related to cloudinit.stages module.""" import os +import stat + +import pytest from cloudinit import stages from cloudinit import sources @@ -341,4 +344,63 @@ class TestInit(CiTestCase): self.init.distro.apply_network_config.assert_called_with( net_cfg, bring_up=True) + +class TestInit_InitializeFilesystem: + """Tests for cloudinit.stages.Init._initialize_filesystem. + + TODO: Expand these tests to cover all of _initialize_filesystem's behavior. + """ + + @pytest.yield_fixture + def init(self, paths): + """A fixture which yields a stages.Init instance with paths and cfg set + + As it is replaced with a mock, consumers of this fixture can set + `init.cfg` if the default empty dict configuration is not appropriate. + """ + with mock.patch( + "cloudinit.stages.Init.cfg", mock.PropertyMock(return_value={}) + ): + with mock.patch("cloudinit.stages.util.ensure_dirs"): + init = stages.Init() + init._paths = paths + yield init + + @mock.patch("cloudinit.stages.util.ensure_file") + def test_ensure_file_not_called_if_no_log_file_configured( + self, m_ensure_file, init + ): + """If no log file is configured, we should not ensure its existence.""" + init.cfg = {} + + init._initialize_filesystem() + + assert 0 == m_ensure_file.call_count + + def test_log_files_existence_is_ensured_if_configured(self, init, tmpdir): + """If a log file is configured, we should ensure its existence.""" + log_file = tmpdir.join("cloud-init.log") + init.cfg = {"def_log_file": str(log_file)} + + init._initialize_filesystem() + + assert log_file.exists + + def test_existing_file_permissions_are_not_modified(self, init, tmpdir): + """If the log file already exists, we should not modify its permissions + + See https://bugs.launchpad.net/cloud-init/+bug/1900837. + """ + # Use a mode that will never be made the default so this test will + # always be valid + mode = 0o606 + log_file = tmpdir.join("cloud-init.log") + log_file.ensure() + log_file.chmod(mode) + init.cfg = {"def_log_file": str(log_file)} + + init._initialize_filesystem() + + assert mode == stat.S_IMODE(log_file.stat().mode) + # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 096a3037..77714928 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -771,4 +771,49 @@ class TestMountCb: ] == callback.call_args_list +@mock.patch("cloudinit.util.write_file") +class TestEnsureFile: + """Tests for ``cloudinit.util.ensure_file``.""" + + def test_parameters_passed_through(self, m_write_file): + """Test the parameters in the signature are passed to write_file.""" + util.ensure_file( + mock.sentinel.path, + mode=mock.sentinel.mode, + preserve_mode=mock.sentinel.preserve_mode, + ) + + assert 1 == m_write_file.call_count + args, kwargs = m_write_file.call_args + assert (mock.sentinel.path,) == args + assert mock.sentinel.mode == kwargs["mode"] + assert mock.sentinel.preserve_mode == kwargs["preserve_mode"] + + @pytest.mark.parametrize( + "kwarg,expected", + [ + # Files should be world-readable by default + ("mode", 0o644), + # The previous behaviour of not preserving mode should be retained + ("preserve_mode", False), + ], + ) + def test_defaults(self, m_write_file, kwarg, expected): + """Test that ensure_file defaults appropriately.""" + util.ensure_file(mock.sentinel.path) + + assert 1 == m_write_file.call_count + _args, kwargs = m_write_file.call_args + assert expected == kwargs[kwarg] + + def test_static_parameters_are_passed(self, m_write_file): + """Test that the static write_files parameters are passed correctly.""" + util.ensure_file(mock.sentinel.path) + + assert 1 == m_write_file.call_count + _args, kwargs = m_write_file.call_args + assert "" == kwargs["content"] + assert "ab" == kwargs["omode"] + + # vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index e47f1cf6..83727544 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1804,8 +1804,12 @@ def append_file(path, content): write_file(path, content, omode="ab", mode=None) -def ensure_file(path, mode=0o644): - write_file(path, content='', omode="ab", mode=mode) +def ensure_file( + path, mode: int = 0o644, *, preserve_mode: bool = False +) -> None: + write_file( + path, content='', omode="ab", mode=mode, preserve_mode=preserve_mode + ) def safe_int(possible_int): -- cgit v1.2.3 From 404f0a4a6542cdc721901d149ac981a81199aa79 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 26 Oct 2020 11:16:20 -0500 Subject: refactor integration testing infrastructure (#610) * Separated IntegrationClient into separate cloud and instance abstractions. This makes it easier to control the lifetime of the pycloudlib's cloud and instance abstractions separately. * Created new cloud-specific subclasses accordingly * Moved platform parsing and initialization code into its own file * Created new session-wide autorun fixture to automatically initialize and destroy the dynamic cloud --- tests/integration_tests/clouds.py | 126 ++++++++++++++++++ tests/integration_tests/conftest.py | 75 +++++++---- tests/integration_tests/instances.py | 157 ++++++++++++++++++++++ tests/integration_tests/platforms.py | 246 ----------------------------------- 4 files changed, 335 insertions(+), 269 deletions(-) create mode 100644 tests/integration_tests/clouds.py create mode 100644 tests/integration_tests/instances.py delete mode 100644 tests/integration_tests/platforms.py diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py new file mode 100644 index 00000000..06f1c623 --- /dev/null +++ b/tests/integration_tests/clouds.py @@ -0,0 +1,126 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from abc import ABC, abstractmethod +import logging + +from pycloudlib import EC2, GCE, Azure, OCI, LXD + +from tests.integration_tests import integration_settings +from tests.integration_tests.instances import ( + IntegrationEc2Instance, + IntegrationGceInstance, + IntegrationAzureInstance, IntegrationInstance, + IntegrationOciInstance, + IntegrationLxdContainerInstance, +) + +try: + from typing import Optional +except ImportError: + pass + + +log = logging.getLogger('integration_testing') + + +class IntegrationCloud(ABC): + datasource = None # type: Optional[str] + integration_instance_cls = IntegrationInstance + + def __init__(self, settings=integration_settings): + self.settings = settings + self.cloud_instance = self._get_cloud_instance() + self.image_id = self._get_initial_image() + + @abstractmethod + def _get_cloud_instance(self): + raise NotImplementedError + + def _get_initial_image(self): + image_id = self.settings.OS_IMAGE + try: + image_id = self.cloud_instance.released_image( + self.settings.OS_IMAGE) + except (ValueError, IndexError): + pass + return image_id + + def launch(self, user_data=None, launch_kwargs=None, + settings=integration_settings): + if self.settings.EXISTING_INSTANCE_ID: + log.info( + 'Not launching instance due to EXISTING_INSTANCE_ID. ' + 'Instance id: %s', self.settings.EXISTING_INSTANCE_ID) + self.instance = self.cloud_instance.get_instance( + self.settings.EXISTING_INSTANCE_ID + ) + return + launch_kwargs = { + 'image_id': self.image_id, + 'user_data': user_data, + 'wait': False, + } + launch_kwargs.update(launch_kwargs) + pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) + pycloudlib_instance.wait(raise_on_cloudinit_failure=False) + log.info('Launched instance: %s', pycloudlib_instance) + return self.get_instance(pycloudlib_instance, settings) + + def get_instance(self, cloud_instance, settings=integration_settings): + return self.integration_instance_cls(self, cloud_instance, settings) + + def destroy(self): + pass + + def snapshot(self, instance): + return self.cloud_instance.snapshot(instance, clean=True) + + +class Ec2Cloud(IntegrationCloud): + datasource = 'ec2' + integration_instance_cls = IntegrationEc2Instance + + def _get_cloud_instance(self): + return EC2(tag='ec2-integration-test') + + +class GceCloud(IntegrationCloud): + datasource = 'gce' + integration_instance_cls = IntegrationGceInstance + + def _get_cloud_instance(self): + return GCE( + tag='gce-integration-test', + project=self.settings.GCE_PROJECT, + region=self.settings.GCE_REGION, + zone=self.settings.GCE_ZONE, + ) + + +class AzureCloud(IntegrationCloud): + datasource = 'azure' + integration_instance_cls = IntegrationAzureInstance + + def _get_cloud_instance(self): + return Azure(tag='azure-integration-test') + + def destroy(self): + self.cloud_instance.delete_resource_group() + + +class OciCloud(IntegrationCloud): + datasource = 'oci' + integration_instance_cls = IntegrationOciInstance + + def _get_cloud_instance(self): + return OCI( + tag='oci-integration-test', + compartment_id=self.settings.OCI_COMPARTMENT_ID + ) + + +class LxdContainerCloud(IntegrationCloud): + datasource = 'lxd_container' + integration_instance_cls = IntegrationLxdContainerInstance + + def _get_cloud_instance(self): + return LXD(tag='lxd-integration-test') diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 9c4eb03a..9163ac66 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -1,21 +1,32 @@ # This file is part of cloud-init. See LICENSE file for license information. -import os import logging +import os import pytest import sys from contextlib import contextmanager from tests.integration_tests import integration_settings -from tests.integration_tests.platforms import ( - dynamic_client, - LxdContainerClient, - client_name_to_class +from tests.integration_tests.clouds import ( + Ec2Cloud, + GceCloud, + AzureCloud, + OciCloud, + LxdContainerCloud, ) + log = logging.getLogger('integration_testing') log.addHandler(logging.StreamHandler(sys.stdout)) log.setLevel(logging.INFO) +platforms = { + 'ec2': Ec2Cloud, + 'gce': GceCloud, + 'azure': AzureCloud, + 'oci': OciCloud, + 'lxd_container': LxdContainerCloud, +} + def pytest_runtest_setup(item): """Skip tests on unsupported clouds. @@ -25,7 +36,7 @@ def pytest_runtest_setup(item): platform, then skip the test. If platform specific marks are not specified, then we assume the test can be run anywhere. """ - all_platforms = client_name_to_class.keys() + all_platforms = platforms.keys() supported_platforms = set(all_platforms).intersection( mark.name for mark in item.iter_markers()) current_platform = integration_settings.PLATFORM @@ -40,33 +51,51 @@ def disable_subp_usage(request): pass +@pytest.yield_fixture(scope='session') +def session_cloud(): + if integration_settings.PLATFORM not in platforms.keys(): + raise ValueError( + "{} is an invalid PLATFORM specified in settings. " + "Must be one of {}".format( + integration_settings.PLATFORM, list(platforms.keys()) + ) + ) + + cloud = platforms[integration_settings.PLATFORM]() + yield cloud + cloud.destroy() + + @pytest.fixture(scope='session', autouse=True) -def setup_image(): +def setup_image(session_cloud): """Setup the target environment with the correct version of cloud-init. So we can launch instances / run tests with the correct image """ - client = dynamic_client() - log.info('Setting up environment for %s', client.datasource) - client.emit_settings_to_log() + client = None + log.info('Setting up environment for %s', session_cloud.datasource) if integration_settings.CLOUD_INIT_SOURCE == 'NONE': pass # that was easy elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': - if not isinstance(client, LxdContainerClient): + if session_cloud.datasource != 'lxd_container': raise ValueError( 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') # The mount needs to happen after the instance is launched, so # no further action needed here elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED': - client.launch() + client = session_cloud.launch() client.install_proposed_image() elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'): - client.launch() + client = session_cloud.launch() client.install_ppa(integration_settings.CLOUD_INIT_SOURCE) elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)): - client.launch() + client = session_cloud.launch() client.install_deb() - if client.instance: + else: + raise ValueError( + 'Invalid value for CLOUD_INIT_SOURCE setting: {}'.format( + integration_settings.CLOUD_INIT_SOURCE)) + if client: # Even if we're keeping instances, we don't want to keep this # one around as it was just for image creation client.destroy() @@ -74,7 +103,7 @@ def setup_image(): @contextmanager -def _client(request, fixture_utils): +def _client(request, fixture_utils, session_cloud): """Fixture implementation for the client fixtures. Launch the dynamic IntegrationClient instance using any provided @@ -82,28 +111,28 @@ def _client(request, fixture_utils): """ user_data = fixture_utils.closest_marker_first_arg_or( request, 'user_data', None) - with dynamic_client(user_data=user_data) as instance: + with session_cloud.launch(user_data=user_data) as instance: yield instance @pytest.yield_fixture -def client(request, fixture_utils): +def client(request, fixture_utils, session_cloud): """Provide a client that runs for every test.""" - with _client(request, fixture_utils) as client: + with _client(request, fixture_utils, session_cloud) as client: yield client @pytest.yield_fixture(scope='module') -def module_client(request, fixture_utils): +def module_client(request, fixture_utils, session_cloud): """Provide a client that runs once per module.""" - with _client(request, fixture_utils) as client: + with _client(request, fixture_utils, session_cloud) as client: yield client @pytest.yield_fixture(scope='class') -def class_client(request, fixture_utils): +def class_client(request, fixture_utils, session_cloud): """Provide a client that runs once per class.""" - with _client(request, fixture_utils) as client: + with _client(request, fixture_utils, session_cloud) as client: yield client diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py new file mode 100644 index 00000000..d64c1ab2 --- /dev/null +++ b/tests/integration_tests/instances.py @@ -0,0 +1,157 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import logging +import os +from tempfile import NamedTemporaryFile + +from pycloudlib.instance import BaseInstance + +import cloudinit +from cloudinit.subp import subp +from tests.integration_tests import integration_settings + +try: + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from tests.integration_tests.clouds import IntegrationCloud +except ImportError: + pass + + +log = logging.getLogger('integration_testing') + + +class IntegrationInstance: + use_sudo = True + + def __init__(self, cloud: 'IntegrationCloud', instance: BaseInstance, + settings=integration_settings): + self.cloud = cloud + self.instance = instance + self.settings = settings + + def emit_settings_to_log(self) -> None: + log.info( + "\n".join( + ["Settings:"] + + [ + "{}={}".format(key, getattr(self.settings, key)) + for key in sorted(self.settings.current_settings) + ] + ) + ) + + def destroy(self): + self.instance.delete() + + def execute(self, command): + return self.instance.execute(command) + + def pull_file(self, remote_file, local_file): + self.instance.pull_file(remote_file, local_file) + + def push_file(self, local_path, remote_path): + self.instance.push_file(local_path, remote_path) + + def read_from_file(self, remote_path) -> str: + tmp_file = NamedTemporaryFile('r') + self.pull_file(remote_path, tmp_file.name) + with tmp_file as f: + contents = f.read() + return contents + + def write_to_file(self, remote_path, contents: str): + # Writes file locally and then pushes it rather + # than writing the file directly on the instance + with NamedTemporaryFile('w', delete=False) as tmp_file: + tmp_file.write(contents) + + try: + self.push_file(tmp_file.name, remote_path) + finally: + os.unlink(tmp_file.name) + + def snapshot(self): + return self.cloud.snapshot(self.instance) + + def _install_new_cloud_init(self, remote_script): + self.execute(remote_script) + version = self.execute('cloud-init -v').split()[-1] + log.info('Installed cloud-init version: %s', version) + self.instance.clean() + image_id = self.snapshot() + log.info('Created new image: %s', image_id) + self.cloud.image_id = image_id + + def install_proposed_image(self): + log.info('Installing proposed image') + remote_script = ( + '{sudo} echo deb "http://archive.ubuntu.com/ubuntu ' + '$(lsb_release -sc)-proposed main" | ' + '{sudo} tee /etc/apt/sources.list.d/proposed.list\n' + '{sudo} apt-get update -q\n' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '') + self._install_new_cloud_init(remote_script) + + def install_ppa(self, repo): + log.info('Installing PPA') + remote_script = ( + '{sudo} add-apt-repository {repo} -y && ' + '{sudo} apt-get update -q && ' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '', repo=repo) + self._install_new_cloud_init(remote_script) + + def install_deb(self): + log.info('Installing deb package') + deb_path = integration_settings.CLOUD_INIT_SOURCE + deb_name = os.path.basename(deb_path) + remote_path = '/var/tmp/{}'.format(deb_name) + self.push_file( + local_path=integration_settings.CLOUD_INIT_SOURCE, + remote_path=remote_path) + remote_script = '{sudo} dpkg -i {path}'.format( + sudo='sudo' if self.use_sudo else '', path=remote_path) + self._install_new_cloud_init(remote_script) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.settings.KEEP_INSTANCE: + self.destroy() + + +class IntegrationEc2Instance(IntegrationInstance): + pass + + +class IntegrationGceInstance(IntegrationInstance): + pass + + +class IntegrationAzureInstance(IntegrationInstance): + pass + + +class IntegrationOciInstance(IntegrationInstance): + pass + + +class IntegrationLxdContainerInstance(IntegrationInstance): + use_sudo = False + + def __init__(self, cloud: 'IntegrationCloud', instance: BaseInstance, + settings=integration_settings): + super().__init__(cloud, instance, settings) + if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + self._mount_source() + + def _mount_source(self): + command = ( + 'lxc config device add {name} host-cloud-init disk ' + 'source={cloudinit_path} ' + 'path=/usr/lib/python3/dist-packages/cloudinit' + ).format( + name=self.instance.name, cloudinit_path=cloudinit.__path__[0]) + subp(command.split()) diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py deleted file mode 100644 index bade5fe0..00000000 --- a/tests/integration_tests/platforms.py +++ /dev/null @@ -1,246 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -from abc import ABC, abstractmethod -import logging -import os -from tempfile import NamedTemporaryFile - -from pycloudlib import EC2, GCE, Azure, OCI, LXD -from pycloudlib.cloud import BaseCloud -from pycloudlib.instance import BaseInstance - -import cloudinit -from cloudinit.subp import subp -from tests.integration_tests import integration_settings - -try: - from typing import Callable, Optional -except ImportError: - pass - - -log = logging.getLogger('integration_testing') - - -class IntegrationClient(ABC): - client = None # type: Optional[BaseCloud] - instance = None # type: Optional[BaseInstance] - datasource = None # type: Optional[str] - use_sudo = True - current_image = None - - def __init__(self, user_data=None, instance_type=None, - settings=integration_settings, launch_kwargs=None): - self.user_data = user_data - self.instance_type = settings.INSTANCE_TYPE if \ - instance_type is None else instance_type - self.settings = settings - self.launch_kwargs = launch_kwargs if launch_kwargs else {} - self.client = self._get_client() - - def emit_settings_to_log(self) -> None: - log.info( - "\n".join( - ["Settings:"] - + [ - "{}={}".format(key, getattr(self.settings, key)) - for key in sorted(self.settings.current_settings) - ] - ) - ) - - @abstractmethod - def _get_client(self): - raise NotImplementedError - - def _get_image(self): - if self.current_image: - return self.current_image - image_id = self.settings.OS_IMAGE - try: - image_id = self.client.released_image(self.settings.OS_IMAGE) - except (ValueError, IndexError): - pass - return image_id - - def launch(self): - if self.settings.EXISTING_INSTANCE_ID: - log.info( - 'Not launching instance due to EXISTING_INSTANCE_ID. ' - 'Instance id: %s', self.settings.EXISTING_INSTANCE_ID) - self.instance = self.client.get_instance( - self.settings.EXISTING_INSTANCE_ID - ) - return - image_id = self._get_image() - launch_args = { - 'image_id': image_id, - 'user_data': self.user_data, - 'wait': False, - } - if self.instance_type: - launch_args['instance_type'] = self.instance_type - launch_args.update(self.launch_kwargs) - self.instance = self.client.launch(**launch_args) - self.instance.wait(raise_on_cloudinit_failure=False) - log.info('Launched instance: %s', self.instance) - - def destroy(self): - self.instance.delete() - - def execute(self, command): - return self.instance.execute(command) - - def pull_file(self, remote_file, local_file): - self.instance.pull_file(remote_file, local_file) - - def push_file(self, local_path, remote_path): - self.instance.push_file(local_path, remote_path) - - def read_from_file(self, remote_path) -> str: - tmp_file = NamedTemporaryFile('r') - self.pull_file(remote_path, tmp_file.name) - with tmp_file as f: - contents = f.read() - return contents - - def write_to_file(self, remote_path, contents: str): - # Writes file locally and then pushes it rather - # than writing the file directly on the instance - with NamedTemporaryFile('w', delete=False) as tmp_file: - tmp_file.write(contents) - - try: - self.push_file(tmp_file.name, remote_path) - finally: - os.unlink(tmp_file.name) - - def snapshot(self): - return self.client.snapshot(self.instance, clean=True) - - def _install_new_cloud_init(self, remote_script): - self.execute(remote_script) - version = self.execute('cloud-init -v').split()[-1] - log.info('Installed cloud-init version: %s', version) - self.instance.clean() - image_id = self.snapshot() - log.info('Created new image: %s', image_id) - IntegrationClient.current_image = image_id - - def install_proposed_image(self): - log.info('Installing proposed image') - remote_script = ( - '{sudo} echo deb "http://archive.ubuntu.com/ubuntu ' - '$(lsb_release -sc)-proposed main" | ' - '{sudo} tee /etc/apt/sources.list.d/proposed.list\n' - '{sudo} apt-get update -q\n' - '{sudo} apt-get install -qy cloud-init' - ).format(sudo='sudo' if self.use_sudo else '') - self._install_new_cloud_init(remote_script) - - def install_ppa(self, repo): - log.info('Installing PPA') - remote_script = ( - '{sudo} add-apt-repository {repo} -y && ' - '{sudo} apt-get update -q && ' - '{sudo} apt-get install -qy cloud-init' - ).format(sudo='sudo' if self.use_sudo else '', repo=repo) - self._install_new_cloud_init(remote_script) - - def install_deb(self): - log.info('Installing deb package') - deb_path = integration_settings.CLOUD_INIT_SOURCE - deb_name = os.path.basename(deb_path) - remote_path = '/var/tmp/{}'.format(deb_name) - self.push_file( - local_path=integration_settings.CLOUD_INIT_SOURCE, - remote_path=remote_path) - remote_script = '{sudo} dpkg -i {path}'.format( - sudo='sudo' if self.use_sudo else '', path=remote_path) - self._install_new_cloud_init(remote_script) - - def __enter__(self): - self.launch() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if not self.settings.KEEP_INSTANCE: - self.destroy() - - -class Ec2Client(IntegrationClient): - datasource = 'ec2' - - def _get_client(self): - return EC2(tag='ec2-integration-test') - - -class GceClient(IntegrationClient): - datasource = 'gce' - - def _get_client(self): - return GCE( - tag='gce-integration-test', - project=self.settings.GCE_PROJECT, - region=self.settings.GCE_REGION, - zone=self.settings.GCE_ZONE, - ) - - -class AzureClient(IntegrationClient): - datasource = 'azure' - - def _get_client(self): - return Azure(tag='azure-integration-test') - - -class OciClient(IntegrationClient): - datasource = 'oci' - - def _get_client(self): - return OCI( - tag='oci-integration-test', - compartment_id=self.settings.OCI_COMPARTMENT_ID - ) - - -class LxdContainerClient(IntegrationClient): - datasource = 'lxd_container' - use_sudo = False - - def _get_client(self): - return LXD(tag='lxd-integration-test') - - def _mount_source(self): - command = ( - 'lxc config device add {name} host-cloud-init disk ' - 'source={cloudinit_path} ' - 'path=/usr/lib/python3/dist-packages/cloudinit' - ).format( - name=self.instance.name, cloudinit_path=cloudinit.__path__[0]) - subp(command.split()) - - def launch(self): - super().launch() - if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': - self._mount_source() - - -client_name_to_class = { - 'ec2': Ec2Client, - 'gce': GceClient, - 'azure': AzureClient, - 'oci': OciClient, - 'lxd_container': LxdContainerClient -} - -try: - dynamic_client = client_name_to_class[ - integration_settings.PLATFORM - ] # type: Callable[..., IntegrationClient] -except KeyError: - raise ValueError( - "{} is an invalid PLATFORM specified in settings. " - "Must be one of {}".format( - integration_settings.PLATFORM, list(client_name_to_class.keys()) - ) - ) -- cgit v1.2.3 From b8bd08194192035a13083539b31cbcaebfe4c577 Mon Sep 17 00:00:00 2001 From: Manuel Aguilera Date: Tue, 27 Oct 2020 07:19:51 -0700 Subject: gentoo: fix hostname rendering when value has a comment (#611) Gentoo's hostname file format instead of being just the host name is hostname=thename". The old code works fine when the file has no comments but if there is a comment the line ``` gentoo_hostname_config = 'hostname="%s"' % conf ``` can render an invalid hostname file that looks similar to ``` hostname="#This is the host namehello" ``` The fix inserts the hostname in a gentoo friendly way so that it gets handled by HostnameConf as a whole and comments are handled and preserved --- cloudinit/distros/gentoo.py | 10 ++++++---- tests/unittests/test_distros/test_gentoo.py | 26 ++++++++++++++++++++++++++ tools/.github-cla-signers | 1 + 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/unittests/test_distros/test_gentoo.py diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 2bee1c89..e9b82602 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -160,10 +160,12 @@ class Distro(distros.Distro): pass if not conf: conf = HostnameConf('') - conf.set_hostname(your_hostname) - gentoo_hostname_config = 'hostname="%s"' % conf - gentoo_hostname_config = gentoo_hostname_config.replace('\n', '') - util.write_file(out_fn, gentoo_hostname_config, 0o644) + + # Many distro's format is the hostname by itself, and that is the + # way HostnameConf works but gentoo expects it to be in + # hostname="the-actual-hostname" + conf.set_hostname('hostname="%s"' % your_hostname) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/tests/unittests/test_distros/test_gentoo.py b/tests/unittests/test_distros/test_gentoo.py new file mode 100644 index 00000000..37a4f51f --- /dev/null +++ b/tests/unittests/test_distros/test_gentoo.py @@ -0,0 +1,26 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import util +from cloudinit import atomic_helper +from cloudinit.tests.helpers import CiTestCase +from . import _get_distro + + +class TestGentoo(CiTestCase): + + def test_write_hostname(self): + distro = _get_distro("gentoo") + hostname = "myhostname" + hostfile = self.tmp_path("hostfile") + distro._write_hostname(hostname, hostfile) + self.assertEqual('hostname="myhostname"\n', util.load_file(hostfile)) + + def test_write_existing_hostname_with_comments(self): + distro = _get_distro("gentoo") + hostname = "myhostname" + contents = '#This is the hostname\nhostname="localhost"' + hostfile = self.tmp_path("hostfile") + atomic_helper.write_file(hostfile, contents, omode="w") + distro._write_hostname(hostname, hostfile) + self.assertEqual('#This is the hostname\nhostname="myhostname"\n', + util.load_file(hostfile)) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d93d0153..b7e9dc02 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -13,6 +13,7 @@ johnsonshi jqueuniet landon912 lucasmoura +manuelisimo marlluslustosa matthewruffell nishigori -- cgit v1.2.3 From f50f635afe5da4e8971ec36719bdc9249cda2d0e Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Tue, 27 Oct 2020 20:54:30 +0000 Subject: Update network config docs to clarify MAC address quoting (#623) Also update MAC addresses used in testcases to remove quotes where not required and add single quotes where quotes are required. --- doc/rtd/topics/network-config-format-v1.rst | 36 ++++++++++++++++++-------- doc/rtd/topics/network-config-format-v2.rst | 13 ++++++++-- tests/unittests/test_net.py | 40 ++++++++++++++--------------- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/doc/rtd/topics/network-config-format-v1.rst b/doc/rtd/topics/network-config-format-v1.rst index dfbde514..92e81897 100644 --- a/doc/rtd/topics/network-config-format-v1.rst +++ b/doc/rtd/topics/network-config-format-v1.rst @@ -64,6 +64,14 @@ structure. The MAC Address is a device unique identifier that most Ethernet-based network devices possess. Specifying a MAC Address is optional. +.. note:: + + MAC addresses must be strings. As MAC addresses which consist of only the + digits 0-9 (i.e. no hex a-f) can be interpreted as a base 60 integer per + the `YAML 1.1 spec`_ it is best practice to quote all MAC addresses to ensure + they are parsed as strings regardless of value. + +.. _YAML 1.1 spec: https://yaml.org/type/int.html .. note:: @@ -91,7 +99,7 @@ packet- or frame-based network. Specifying ``mtu`` is optional. # Simple network adapter - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' # Second nic with Jumbo frames - type: physical name: jumbo0 @@ -124,6 +132,14 @@ bond interfaces. Specifying a MAC Address is optional. If ``mac_address`` is not present, then the bond will use one of the MAC Address values from one of the bond interfaces. +.. note:: + + MAC addresses must be strings. As MAC addresses which consist of only the + digits 0-9 (i.e. no hex a-f) can be interpreted as a base 60 integer per + the `YAML 1.1 spec`_ it is best practice to quote all MAC addresses to ensure + they are parsed as strings regardless of value. + +.. _YAML 1.1 spec: https://yaml.org/type/int.html **bond_interfaces**: ** @@ -194,7 +210,7 @@ Valid ``params`` keys are: # Simple network adapter - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' # 10G pair - type: physical name: gbe0 @@ -246,7 +262,7 @@ Valid keys are: # Simple network adapter - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' # Second nic with Jumbo frames - type: physical name: jumbo0 @@ -303,7 +319,7 @@ packet- or frame-based network. Specifying ``mtu`` is optional. # Physical interfaces. - type: physical name: eth0 - mac_address: "c0:d6:9f:2c:e8:80" + mac_address: c0:d6:9f:2c:e8:80 # VLAN interface. - type: vlan name: eth0.101 @@ -327,7 +343,7 @@ the following keys: config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: static address: 192.168.23.14/27 @@ -358,7 +374,7 @@ has the following keys: config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: static address: 192.168.23.14/24 @@ -410,7 +426,7 @@ the subnet dictionary. config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: dhcp @@ -422,7 +438,7 @@ the subnet dictionary. config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: static address: 192.168.23.14/27 @@ -443,7 +459,7 @@ using the static subnet configuration. config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: dhcp - type: static @@ -462,7 +478,7 @@ using the static subnet configuration. config: - type: physical name: interface0 - mac_address: 00:11:22:33:44:55 + mac_address: '00:11:22:33:44:55' subnets: - type: dhcp - type: static diff --git a/doc/rtd/topics/network-config-format-v2.rst b/doc/rtd/topics/network-config-format-v2.rst index c93e29be..aa17bef5 100644 --- a/doc/rtd/topics/network-config-format-v2.rst +++ b/doc/rtd/topics/network-config-format-v2.rst @@ -94,7 +94,16 @@ NetworkManager does not. **macaddress**: *<(scalar)>* -Device's MAC address in the form "XX:XX:XX:XX:XX:XX". Globs are not allowed. +Device's MAC address in the form XX:XX:XX:XX:XX:XX. Globs are not allowed. + +.. note:: + + MAC addresses must be strings. As MAC addresses which consist of only the + digits 0-9 (i.e. no hex a-f) can be interpreted as a base 60 integer per + the `YAML 1.1 spec`_ it is best practice to quote all MAC addresses to ensure + they are parsed as strings regardless of value. + +.. _YAML 1.1 spec: https://yaml.org/type/int.html **driver**: *<(scalar)>* @@ -458,7 +467,7 @@ This is a complex example which shows most available features: :: # opaque ID for physical interfaces, only referred to by other stanzas id0: match: - macaddress: 00:11:22:33:44:55 + macaddress: '00:11:22:33:44:55' wakeonlan: true dhcp4: true addresses: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 207e47bb..642e60cc 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -910,7 +910,7 @@ NETWORK_CONFIGS = { # Physical interfaces. - type: physical name: eth99 - mac_address: "c0:d6:9f:2c:e8:80" + mac_address: c0:d6:9f:2c:e8:80 subnets: - type: dhcp4 - type: static @@ -926,7 +926,7 @@ NETWORK_CONFIGS = { metric: 10000 - type: physical name: eth1 - mac_address: "cf:d6:af:48:e8:80" + mac_address: cf:d6:af:48:e8:80 - type: nameserver address: - 1.2.3.4 @@ -1743,26 +1743,26 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true # Physical interfaces. - type: physical name: eth0 - mac_address: "c0:d6:9f:2c:e8:80" + mac_address: c0:d6:9f:2c:e8:80 - type: physical name: eth1 - mac_address: "aa:d6:9f:2c:e8:80" + mac_address: aa:d6:9f:2c:e8:80 - type: physical name: eth2 - mac_address: "c0:bb:9f:2c:e8:80" + mac_address: c0:bb:9f:2c:e8:80 - type: physical name: eth3 - mac_address: "66:bb:9f:2c:e8:80" + mac_address: 66:bb:9f:2c:e8:80 - type: physical name: eth4 - mac_address: "98:bb:9f:2c:e8:80" + mac_address: 98:bb:9f:2c:e8:80 # specify how ifupdown should treat iface # control is one of ['auto', 'hotplug', 'manual'] # with manual meaning ifup/ifdown should not affect the iface # useful for things like iscsi root + dhcp - type: physical name: eth5 - mac_address: "98:bb:9f:2c:e8:8a" + mac_address: 98:bb:9f:2c:e8:8a subnets: - type: dhcp control: manual @@ -1793,7 +1793,7 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true name: bond0 # if 'mac_address' is omitted, the MAC is taken from # the first slave. - mac_address: "aa:bb:cc:dd:ee:ff" + mac_address: aa:bb:cc:dd:ee:ff bond_interfaces: - eth1 - eth2 @@ -1888,13 +1888,13 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true config: - type: physical name: bond0s0 - mac_address: "aa:bb:cc:dd:e8:00" + mac_address: aa:bb:cc:dd:e8:00 - type: physical name: bond0s1 - mac_address: "aa:bb:cc:dd:e8:01" + mac_address: aa:bb:cc:dd:e8:01 - type: bond name: bond0 - mac_address: "aa:bb:cc:dd:e8:ff" + mac_address: aa:bb:cc:dd:e8:ff mtu: 9000 bond_interfaces: - bond0s0 @@ -2042,12 +2042,12 @@ iface bond0 inet6 static eth0: match: driver: "virtio_net" - macaddress: "aa:bb:cc:dd:e8:00" + macaddress: aa:bb:cc:dd:e8:00 vf0: set-name: vf0 match: driver: "e1000" - macaddress: "aa:bb:cc:dd:e8:01" + macaddress: aa:bb:cc:dd:e8:01 bonds: bond0: addresses: @@ -2221,7 +2221,7 @@ iface bond0 inet6 static config: - type: physical name: en0 - mac_address: "aa:bb:cc:dd:e8:00" + mac_address: aa:bb:cc:dd:e8:00 - type: vlan mtu: 2222 name: en0.99 @@ -2294,13 +2294,13 @@ iface bond0 inet6 static config: - type: physical name: eth0 - mac_address: "52:54:00:12:34:00" + mac_address: '52:54:00:12:34:00' subnets: - type: static address: 2001:1::100/96 - type: physical name: eth1 - mac_address: "52:54:00:12:34:01" + mac_address: '52:54:00:12:34:01' subnets: - type: static address: 2001:1::101/96 @@ -2385,7 +2385,7 @@ iface bond0 inet6 static config: - type: physical name: eth0 - mac_address: "52:54:00:12:34:00" + mac_address: '52:54:00:12:34:00' subnets: - type: static address: 192.168.1.2/24 @@ -2393,12 +2393,12 @@ iface bond0 inet6 static - type: physical name: eth1 mtu: 1480 - mac_address: "52:54:00:12:34:aa" + mac_address: 52:54:00:12:34:aa subnets: - type: manual - type: physical name: eth2 - mac_address: "52:54:00:12:34:ff" + mac_address: 52:54:00:12:34:ff subnets: - type: manual control: manual -- cgit v1.2.3 From 29f0d136e24826a9b1c2c541755c64c4e51d5847 Mon Sep 17 00:00:00 2001 From: Markus Schade Date: Wed, 28 Oct 2020 14:12:07 +0100 Subject: lp-to-git-users: adding asciiprod (#629) Mapped from lp-markusschade --- tools/.lp-to-git-user | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/.lp-to-git-user b/tools/.lp-to-git-user index 89422dbb..21171ac6 100644 --- a/tools/.lp-to-git-user +++ b/tools/.lp-to-git-user @@ -19,6 +19,7 @@ "larsks": "larsks", "legovini": "paride", "louis": "karibou", + "lp-markusschade": "asciiprod", "madhuri-rai07": "madhuri-rai07", "momousta": "Moustafa-Moustafa", "otubo": "otubo", @@ -30,4 +31,4 @@ "trstringer": "trstringer", "vtqanh": "anhvoms", "xiaofengw": "xiaofengw-vmware" -} \ No newline at end of file +} -- cgit v1.2.3 From f99d4f96b00a9cfec1c721d364cbfd728674e5dc Mon Sep 17 00:00:00 2001 From: Aman306 <45781773+Aman306@users.noreply.github.com> Date: Wed, 28 Oct 2020 23:36:09 +0530 Subject: Add config modules for controlling IBM PowerVM RMC. (#584) Reliable Scalable Cluster Technology (RSCT) is a set of software components that together provide a comprehensive clustering environment(RAS features) for IBM PowerVM based virtual machines. RSCT includes the Resource Monitoring and Control (RMC) subsystem. RMC is a generalized framework used for managing, monitoring, and manipulating resources. RMC runs as a daemon process on individual machines and needs creation of unique node id and restarts during VM boot. LP: #1895979 Co-authored-by: Scott Moser --- cloudinit/config/cc_refresh_rmc_and_interface.py | 159 +++++++++++++++++++++ cloudinit/config/cc_reset_rmc.py | 143 ++++++++++++++++++ config/cloud.cfg.tmpl | 2 + .../test_handler_refresh_rmc_and_interface.py | 109 ++++++++++++++ tools/.github-cla-signers | 1 + 5 files changed, 414 insertions(+) create mode 100644 cloudinit/config/cc_refresh_rmc_and_interface.py create mode 100644 cloudinit/config/cc_reset_rmc.py create mode 100644 tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py new file mode 100644 index 00000000..146758ad --- /dev/null +++ b/cloudinit/config/cc_refresh_rmc_and_interface.py @@ -0,0 +1,159 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + +""" +Refresh IPv6 interface and RMC +------------------------------ +**Summary:** Ensure Network Manager is not managing IPv6 interface + +This module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components +that together provide a comprehensive clustering environment(RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource +Monitoring and Control (RMC) subsystem. RMC is a generalized framework used +for managing, monitoring, and manipulating resources. RMC runs as a daemon +process on individual machines and needs creation of unique node id and +restarts during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- Refreshing RMC +- Disabling NetworkManager from handling IPv6 interface, as IPv6 interface + is used for communication between RMC daemon and PowerVM hypervisor. + +**Internal name:** ``cc_refresh_rmc_and_interface`` + +**Module frequency:** per always + +**Supported distros:** RHEL + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util +from cloudinit import subp +from cloudinit import netinfo + +import errno + +frequency = PER_ALWAYS + +LOG = logging.getLogger(__name__) +# Ensure that /opt/rsct/bin has been added to standard PATH of the +# distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl . +RMCCTRL = 'rmcctrl' + + +def handle(name, _cfg, _cloud, _log, _args): + if not subp.which(RMCCTRL): + LOG.debug("No '%s' in path, disabled", RMCCTRL) + return + + LOG.debug( + 'Making the IPv6 up explicitly. ' + 'Ensuring IPv6 interface is not being handled by NetworkManager ' + 'and it is restarted to re-establish the communication with ' + 'the hypervisor') + + ifaces = find_ipv6_ifaces() + + # Setting NM_CONTROLLED=no for IPv6 interface + # making it down and up + + if len(ifaces) == 0: + LOG.debug("Did not find any interfaces with ipv6 addresses.") + else: + for iface in ifaces: + refresh_ipv6(iface) + disable_ipv6(sysconfig_path(iface)) + restart_network_manager() + + +def find_ipv6_ifaces(): + info = netinfo.netdev_info() + ifaces = [] + for iface, data in info.items(): + if iface == "lo": + LOG.debug('Skipping localhost interface') + if len(data.get("ipv4", [])) != 0: + # skip this interface, as it has ipv4 addrs + continue + ifaces.append(iface) + return ifaces + + +def refresh_ipv6(interface): + # IPv6 interface is explicitly brought up, subsequent to which the + # RMC services are restarted to re-establish the communication with + # the hypervisor. + subp.subp(['ip', 'link', 'set', interface, 'down']) + subp.subp(['ip', 'link', 'set', interface, 'up']) + + +def sysconfig_path(iface): + return '/etc/sysconfig/network-scripts/ifcfg-' + iface + + +def restart_network_manager(): + subp.subp(['systemctl', 'restart', 'NetworkManager']) + + +def disable_ipv6(iface_file): + # Ensuring that the communication b/w the hypervisor and VM is not + # interrupted due to NetworkManager. For this purpose, as part of + # this function, the NM_CONTROLLED is explicitly set to No for IPV6 + # interface and NetworkManager is restarted. + try: + contents = util.load_file(iface_file) + except IOError as e: + if e.errno == errno.ENOENT: + LOG.debug("IPv6 interface file %s does not exist\n", + iface_file) + else: + raise e + + if 'IPV6INIT' not in contents: + LOG.debug("Interface file %s did not have IPV6INIT", iface_file) + return + + LOG.debug("Editing interface file %s ", iface_file) + + # Dropping any NM_CONTROLLED or IPV6 lines from IPv6 interface file. + lines = contents.splitlines() + lines = [line for line in lines if not search(line)] + lines.append("NM_CONTROLLED=no") + + with open(iface_file, "w") as fp: + fp.write("\n".join(lines) + "\n") + + +def search(contents): + # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file. + return( + contents.startswith("IPV6ADDR") or + contents.startswith("IPADDR6") or + contents.startswith("IPV6INIT") or + contents.startswith("NM_CONTROLLED")) + + +def refresh_rmc(): + # To make a healthy connection between RMC daemon and hypervisor we + # refresh RMC. With refreshing RMC we are ensuring that making IPv6 + # down and up shouldn't impact communication between RMC daemon and + # hypervisor. + # -z : stop Resource Monitoring & Control subsystem and all resource + # managers, but the command does not return control to the user + # until the subsystem and all resource managers are stopped. + # -s : start Resource Monitoring & Control subsystem. + try: + subp.subp([RMCCTRL, '-z']) + subp.subp([RMCCTRL, '-s']) + except Exception: + util.logexc(LOG, 'Failed to refresh the RMC subsystem.') + raise diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py new file mode 100644 index 00000000..1cd72774 --- /dev/null +++ b/cloudinit/config/cc_reset_rmc.py @@ -0,0 +1,143 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + + +""" +Reset RMC +------------ +**Summary:** reset rsct node id + +Reset RMC module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components, +that together provide a comprehensive clustering environment (RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource monitoring +and control (RMC) subsystem. RMC is a generalized framework used for managing, +monitoring, and manipulating resources. RMC runs as a daemon process on +individual machines and needs creation of unique node id and restarts +during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- creation of the unique RSCT node id to every instance/virtual machine + and ensure once set, it isn't changed subsequently by cloud-init. + In order to do so, it restarts RSCT service. + +Prerequisite of using this module is to install RSCT packages. + +**Internal name:** ``cc_reset_rmc`` + +**Module frequency:** per instance + +**Supported distros:** rhel, sles and ubuntu + +""" +import os + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util +from cloudinit import subp + +frequency = PER_INSTANCE + +# RMCCTRL is expected to be in system PATH (/opt/rsct/bin) +# The symlink for RMCCTRL and RECFGCT are +# /usr/sbin/rsct/bin/rmcctrl and +# /usr/sbin/rsct/install/bin/recfgct respectively. +RSCT_PATH = '/opt/rsct/install/bin' +RMCCTRL = 'rmcctrl' +RECFGCT = 'recfgct' + +LOG = logging.getLogger(__name__) + +NODE_ID_FILE = '/etc/ct_node_id' + + +def handle(name, _cfg, cloud, _log, _args): + # Ensuring node id has to be generated only once during first boot + if cloud.datasource.platform_type == 'none': + LOG.debug('Skipping creation of new ct_node_id node') + return + + if not os.path.isdir(RSCT_PATH): + LOG.debug("module disabled, RSCT_PATH not present") + return + + orig_path = os.environ.get('PATH') + try: + add_path(orig_path) + reset_rmc() + finally: + if orig_path: + os.environ['PATH'] = orig_path + else: + del os.environ['PATH'] + + +def reconfigure_rsct_subsystems(): + # Reconfigure the RSCT subsystems, which includes removing all RSCT data + # under the /var/ct directory, generating a new node ID, and making it + # appear as if the RSCT components were just installed + try: + out = subp.subp([RECFGCT])[0] + LOG.debug(out.strip()) + return out + except subp.ProcessExecutionError: + util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.') + raise + + +def get_node_id(): + try: + fp = util.load_file(NODE_ID_FILE) + node_id = fp.split('\n')[0] + return node_id + except Exception: + util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE) + raise + + +def add_path(orig_path): + # Adding the RSCT_PATH to env standard path + # So thet cloud init automatically find and + # run RECFGCT to create new node_id. + suff = ":" + orig_path if orig_path else "" + os.environ['PATH'] = RSCT_PATH + suff + return os.environ['PATH'] + + +def rmcctrl(): + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + try: + return subp.subp([RMCCTRL, '-z']) + except Exception: + util.logexc(LOG, 'Failed to stop the RMC subsystem.') + raise + + +def reset_rmc(): + LOG.debug('Attempting to reset RMC.') + + node_id_before = get_node_id() + LOG.debug('Node ID at beginning of module: %s', node_id_before) + + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + rmcctrl() + reconfigure_rsct_subsystems() + + node_id_after = get_node_id() + LOG.debug('Node ID at end of module: %s', node_id_after) + + # Check if new node ID is generated or not + # by comparing old and new node ID + if node_id_after == node_id_before: + msg = 'New node ID did not get generated.' + LOG.error(msg) + raise Exception(msg) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 2beb9b0c..7171aaa5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -135,6 +135,8 @@ cloud_final_modules: - chef - mcollective - salt-minion + - reset_rmc + - refresh_rmc_and_interface - rightscale_userdata - scripts-vendor - scripts-per-once diff --git a/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py new file mode 100644 index 00000000..e13b7793 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py @@ -0,0 +1,109 @@ +from cloudinit.config import cc_refresh_rmc_and_interface as ccrmci + +from cloudinit import util + +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock + +from textwrap import dedent +import logging + +LOG = logging.getLogger(__name__) +MPATH = "cloudinit.config.cc_refresh_rmc_and_interface" +NET_INFO = { + 'lo': {'ipv4': [{'ip': '127.0.0.1', + 'bcast': '', 'mask': '255.0.0.0', + 'scope': 'host'}], + 'ipv6': [{'ip': '::1/128', + 'scope6': 'host'}], 'hwaddr': '', + 'up': 'True'}, + 'env2': {'ipv4': [{'ip': '8.0.0.19', + 'bcast': '8.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8220/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:20', + 'up': 'True'}, + 'env3': {'ipv4': [{'ip': '90.0.0.14', + 'bcast': '90.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8221/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:21', + 'up': 'True'}, + 'env4': {'ipv4': [{'ip': '9.114.23.7', + 'bcast': '9.114.23.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8222/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:22', + 'up': 'True'}, + 'env5': {'ipv4': [], + 'ipv6': [{'ip': 'fe80::9c26:c3ff:fea4:62c8/64', + 'scope6': 'link'}], 'hwaddr': '42:20:86:df:fa:4c', + 'up': 'True'}} + + +class TestRsctNodeFile(t_help.CiTestCase): + def test_disable_ipv6_interface(self): + """test parsing of iface files.""" + fname = self.tmp_path("iface-eth5") + util.write_file(fname, dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + IPV6INIT=yes + IPADDR6=fe80::9c26:c3ff:fea4:62c8/64 + IPV6ADDR=fe80::9c26:c3ff:fea4:62c8/64 + NM_CONTROLLED=yes + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + """)) + + ccrmci.disable_ipv6(fname) + self.assertEqual(dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + NM_CONTROLLED=no + """), util.load_file(fname)) + + @mock.patch(MPATH + '.refresh_rmc') + @mock.patch(MPATH + '.restart_network_manager') + @mock.patch(MPATH + '.disable_ipv6') + @mock.patch(MPATH + '.refresh_ipv6') + @mock.patch(MPATH + '.netinfo.netdev_info') + @mock.patch(MPATH + '.subp.which') + def test_handle(self, m_refresh_rmc, + m_netdev_info, m_refresh_ipv6, m_disable_ipv6, + m_restart_nm, m_which): + """Basic test of handle.""" + m_netdev_info.return_value = NET_INFO + m_which.return_value = '/opt/rsct/bin/rmcctrl' + ccrmci.handle( + "refresh_rmc_and_interface", None, None, None, None) + self.assertEqual(1, m_netdev_info.call_count) + m_refresh_ipv6.assert_called_with('env5') + m_disable_ipv6.assert_called_with( + '/etc/sysconfig/network-scripts/ifcfg-env5') + self.assertEqual(1, m_restart_nm.call_count) + self.assertEqual(1, m_refresh_rmc.call_count) + + @mock.patch(MPATH + '.netinfo.netdev_info') + def test_find_ipv6(self, m_netdev_info): + """find_ipv6_ifaces parses netdev_info returning those with ipv6""" + m_netdev_info.return_value = NET_INFO + found = ccrmci.find_ipv6_ifaces() + self.assertEqual(['env5'], found) + + @mock.patch(MPATH + '.subp.subp') + def test_refresh_ipv6(self, m_subp): + """refresh_ipv6 should ip down and up the interface.""" + iface = "myeth0" + ccrmci.refresh_ipv6(iface) + m_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', iface, 'down']), + mock.call(['ip', 'link', 'set', iface, 'up'])]) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b7e9dc02..475f0872 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,5 +1,6 @@ ader1990 AlexBaranowski +Aman306 beezly bipinbachhao BirknerAlex -- cgit v1.2.3 From 3c432b32de1bdce2699525201396a8bbc6a41f3e Mon Sep 17 00:00:00 2001 From: Lukas Märdian Date: Thu, 29 Oct 2020 14:38:56 +0100 Subject: get_interfaces: don't exclude Open vSwitch bridge/bond members (#608) If an OVS bridge was used as the only/primary interface, the 'init' stage failed with a "Not all expected physical devices present" error, leaving the system with a broken SSH setup. LP: #1898997 --- cloudinit/net/__init__.py | 15 +++++++++++++-- cloudinit/net/tests/test_init.py | 36 +++++++++++++++++++++++++++++++++++- tools/.github-cla-signers | 1 + 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 75e79ca8..de65e7af 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -124,6 +124,15 @@ def master_is_bridge_or_bond(devname): return (os.path.exists(bonding_path) or os.path.exists(bridge_path)) +def master_is_openvswitch(devname): + """Return a bool indicating if devname's master is openvswitch""" + master_path = get_master(devname) + if master_path is None: + return False + ovs_path = sys_dev_path(devname, path="upper_ovs-system") + return os.path.exists(ovs_path) + + def is_netfailover(devname, driver=None): """ netfailover driver uses 3 nics, master, primary and standby. this returns True if the device is either the primary or standby @@ -862,8 +871,10 @@ def get_interfaces(blacklist_drivers=None) -> list: continue if is_bond(name): continue - if get_master(name) is not None and not master_is_bridge_or_bond(name): - continue + if get_master(name) is not None: + if (not master_is_bridge_or_bond(name) and + not master_is_openvswitch(name)): + continue if is_netfailover(name): continue mac = get_interface_mac(name) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 311ab6f8..0535387a 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -190,6 +190,28 @@ class TestReadSysNet(CiTestCase): self.assertTrue(net.master_is_bridge_or_bond('eth1')) self.assertTrue(net.master_is_bridge_or_bond('eth2')) + def test_master_is_openvswitch(self): + ovs_mac = 'bb:cc:aa:bb:cc:aa' + + # No master => False + write_file(os.path.join(self.sysdir, 'eth1', 'address'), ovs_mac) + + self.assertFalse(net.master_is_bridge_or_bond('eth1')) + + # masters without ovs-system => False + write_file(os.path.join(self.sysdir, 'ovs-system', 'address'), ovs_mac) + + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth1', + 'master')) + + self.assertFalse(net.master_is_openvswitch('eth1')) + + # masters with ovs-system => True + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth1', + 'upper_ovs-system')) + + self.assertTrue(net.master_is_openvswitch('eth1')) + def test_is_vlan(self): """is_vlan is True when /sys/net/devname/uevent has DEVTYPE=vlan.""" ensure_file(os.path.join(self.sysdir, 'eth0', 'uevent')) @@ -465,20 +487,32 @@ class TestGetInterfaceMAC(CiTestCase): ): bridge_mac = 'aa:bb:cc:aa:bb:cc' bond_mac = 'cc:bb:aa:cc:bb:aa' + ovs_mac = 'bb:cc:aa:bb:cc:aa' + write_file(os.path.join(self.sysdir, 'br0', 'address'), bridge_mac) write_file(os.path.join(self.sysdir, 'br0', 'bridge'), '') write_file(os.path.join(self.sysdir, 'bond0', 'address'), bond_mac) write_file(os.path.join(self.sysdir, 'bond0', 'bonding'), '') + write_file(os.path.join(self.sysdir, 'ovs-system', 'address'), + ovs_mac) + write_file(os.path.join(self.sysdir, 'eth1', 'address'), bridge_mac) os.symlink('../br0', os.path.join(self.sysdir, 'eth1', 'master')) write_file(os.path.join(self.sysdir, 'eth2', 'address'), bond_mac) os.symlink('../bond0', os.path.join(self.sysdir, 'eth2', 'master')) + write_file(os.path.join(self.sysdir, 'eth3', 'address'), ovs_mac) + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth3', + 'master')) + os.symlink('../ovs-system', os.path.join(self.sysdir, 'eth3', + 'upper_ovs-system')) + interface_names = [interface[0] for interface in net.get_interfaces()] - self.assertEqual(['eth1', 'eth2'], sorted(interface_names)) + self.assertEqual(['eth1', 'eth2', 'eth3', 'ovs-system'], + sorted(interface_names)) class TestInterfaceHasOwnMAC(CiTestCase): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 475f0872..2d81b700 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -20,6 +20,7 @@ matthewruffell nishigori omBratteng onitake +slyon smoser sshedi TheRealFalcon -- cgit v1.2.3 From b46e4a8cff667c8441622089cf7d57aeb88220cd Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Thu, 29 Oct 2020 15:05:42 +0100 Subject: Explicit set IPV6_AUTOCONF and IPV6_FORCE_ACCEPT_RA on static6 (#634) The static and static6 subnet types for network_data.json were being ignored by the Openstack handler, this would cause the code to break and not function properly. As of today, if a static6 configuration is chosen, the interface will still eventually be available to receive router advertisements or be set from NetworkManager to wait for them and cycle the interface in negative case. It is safe to assume that if the interface is manually configured to use static ipv6 address, there's no need to wait for router advertisements. This patch will set automatically IPV6_AUTOCONF and IPV6_FORCE_ACCEPT_RA both to "no" in this case. This patch fixes the specific behavior only for RHEL flavor and sysconfig renderer. It also introduces new unit tests for the specific case as well as adjusts some existent tests to be compatible with the new options. This patch also addresses this problem by assigning the appropriate subnet type for each case on the openstack handler. rhbz: #1889635 rhbz: #1889635 Signed-off-by: Eduardo Otubo otubo@redhat.com --- cloudinit/net/network_state.py | 3 +- cloudinit/net/sysconfig.py | 4 ++ cloudinit/sources/helpers/openstack.py | 8 ++- tests/unittests/test_distros/test_netconfig.py | 2 + tests/unittests/test_net.py | 98 ++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index b2f7d31e..d9e7fd58 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -820,7 +820,8 @@ def _normalize_subnet(subnet): if subnet.get('type') in ('static', 'static6'): normal_subnet.update( - _normalize_net_keys(normal_subnet, address_keys=('address',))) + _normalize_net_keys(normal_subnet, address_keys=( + 'address', 'ip_address',))) normal_subnet['routes'] = [_normalize_route(r) for r in subnet.get('routes', [])] diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index e9337b12..b0eecc44 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -463,6 +463,10 @@ class Renderer(renderer.Renderer): iface_cfg[mtu_key] = subnet['mtu'] else: iface_cfg[mtu_key] = subnet['mtu'] + + if subnet_is_ipv6(subnet) and flavor == 'rhel': + iface_cfg['IPV6_FORCE_ACCEPT_RA'] = False + iface_cfg['IPV6_AUTOCONF'] = False elif subnet_type == 'manual': if flavor == 'suse': LOG.debug('Unknown subnet type setting "%s"', subnet_type) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 65e020c5..3e6365f1 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -602,11 +602,17 @@ def convert_net_json(network_json=None, known_macs=None): elif network['type'] in ['ipv6_slaac', 'ipv6_dhcpv6-stateless', 'ipv6_dhcpv6-stateful']: subnet.update({'type': network['type']}) - elif network['type'] in ['ipv4', 'ipv6']: + elif network['type'] in ['ipv4', 'static']: subnet.update({ 'type': 'static', 'address': network.get('ip_address'), }) + elif network['type'] in ['ipv6', 'static6']: + cfg.update({'accept-ra': False}) + subnet.update({ + 'type': 'static6', + 'address': network.get('ip_address'), + }) # Enable accept_ra for stateful and legacy ipv6_dhcp types if network['type'] in ['ipv6_dhcpv6-stateful', 'ipv6_dhcp']: diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 3f3fe3eb..a1df066a 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -514,7 +514,9 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase): DEVICE=eth0 IPV6ADDR=2607:f0d0:1002:0011::2/64 IPV6INIT=yes + IPV6_AUTOCONF=no IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 + IPV6_FORCE_ACCEPT_RA=no NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 642e60cc..5af82e20 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -752,7 +752,9 @@ IPADDR=172.19.1.34 IPV6ADDR=2001:DB8::10/64 IPV6ADDR_SECONDARIES="2001:DB9::10/64 2001:DB10::10/64" IPV6INIT=yes +IPV6_AUTOCONF=no IPV6_DEFAULTGW=2001:DB8::1 +IPV6_FORCE_ACCEPT_RA=no NETMASK=255.255.252.0 NM_CONTROLLED=no ONBOOT=yes @@ -1027,6 +1029,8 @@ NETWORK_CONFIGS = { IPADDR=192.168.14.2 IPV6ADDR=2001:1::1/64 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no NETMASK=255.255.255.0 NM_CONTROLLED=no ONBOOT=yes @@ -1253,6 +1257,33 @@ NETWORK_CONFIGS = { """), }, }, + 'static6': { + 'yaml': textwrap.dedent("""\ + version: 1 + config: + - type: 'physical' + name: 'iface0' + accept-ra: 'no' + subnets: + - type: 'static6' + address: 2001:1::1/64 + """).rstrip(' '), + 'expected_sysconfig_rhel': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=iface0 + IPV6ADDR=2001:1::1/64 + IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no + DEVICE=iface0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + }, + }, 'dhcpv6_stateless': { 'expected_eni': textwrap.dedent("""\ auto lo @@ -1643,6 +1674,8 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true IPADDR=192.168.14.2 IPV6ADDR=2001:1::1/64 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no IPV6_DEFAULTGW=2001:4800:78ff:1b::1 MACADDR=bb:bb:bb:bb:bb:aa NETMASK=255.255.255.0 @@ -2172,6 +2205,8 @@ iface bond0 inet6 static IPADDR1=192.168.1.2 IPV6ADDR=2001:1::1/92 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no MTU=9000 NETMASK=255.255.255.0 NETMASK1=255.255.255.0 @@ -2277,6 +2312,8 @@ iface bond0 inet6 static IPADDR1=192.168.1.2 IPV6ADDR=2001:1::bbbb/96 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no IPV6_DEFAULTGW=2001:1::1 MTU=2222 NETMASK=255.255.255.0 @@ -2360,6 +2397,8 @@ iface bond0 inet6 static HWADDR=52:54:00:12:34:00 IPV6ADDR=2001:1::100/96 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet @@ -2372,6 +2411,8 @@ iface bond0 inet6 static HWADDR=52:54:00:12:34:01 IPV6ADDR=2001:1::101/96 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet @@ -3178,6 +3219,61 @@ USERCTL=no self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) + def test_stattic6_from_json(self): + net_json = { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }, { + "netmask": "0.0.0.0", # A second default gateway + "network": "0.0.0.0", + "gateway": "172.20.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }, { + "network_id": "mgmt", + "netmask": "ffff:ffff:ffff:ffff::", + "link": "interface1", + "mode": "link-local", + "routes": [], + "ip_address": "fe80::c096:67ff:fe5c:6e84", + "type": "static6", + "id": "network1", + "services": [], + "accept-ra": "false" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + } + macs = {'fa:16:3e:ed:9a:59': 'eth0'} + render_dir = self.tmp_dir() + network_cfg = openstack.convert_net_json(net_json, known_macs=macs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = self._get_renderer() + with self.assertRaises(ValueError): + renderer.render_network_state(ns, target=render_dir) + self.assertEqual([], os.listdir(render_dir)) + + def test_static6_from_yaml(self): + entry = NETWORK_CONFIGS['static6'] + found = self._render_and_read(network_config=yaml.load( + entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + def test_dhcpv6_reject_ra_config_v2(self): entry = NETWORK_CONFIGS['dhcpv6_reject_ra'] found = self._render_and_read(network_config=yaml.load( @@ -3295,6 +3391,8 @@ USERCTL=no IPADDR=192.168.42.100 IPV6ADDR=2001:db8::100/32 IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no IPV6_DEFAULTGW=2001:db8::1 NETMASK=255.255.255.0 NM_CONTROLLED=no -- cgit v1.2.3 From 1431c8a1bddaabf85e1bbb32bf316a3aef20036e Mon Sep 17 00:00:00 2001 From: Markus Schade Date: Thu, 29 Oct 2020 15:45:47 +0100 Subject: Hetzner: initialize instance_id from system-serial-number (#630) Hetzner Cloud also provides the instance ID in SMBIOS information. Use it to locally check_instance_id and to compared with instance_id from metadata service. LP: #1885527 --- cloudinit/sources/DataSourceHetzner.py | 36 +++++++++++++++++++++---- tests/unittests/test_datasource/test_hetzner.py | 19 ++++++------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 79353882..1d965bf7 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -3,15 +3,18 @@ # # This file is part of cloud-init. See LICENSE file for license information. # -"""Hetzner Cloud API Documentation. +"""Hetzner Cloud API Documentation https://docs.hetzner.cloud/""" +from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import sources from cloudinit import util import cloudinit.sources.helpers.hetzner as hc_helper +LOG = logging.getLogger(__name__) + BASE_URL_V1 = 'http://169.254.169.254/hetzner/v1' BUILTIN_DS_CONFIG = { @@ -43,9 +46,12 @@ class DataSourceHetzner(sources.DataSource): self._network_config = None self.dsmode = sources.DSMODE_NETWORK - def get_data(self): - if not on_hetzner(): + def _get_data(self): + (on_hetzner, serial) = get_hcloud_data() + + if not on_hetzner: return False + nic = cloudnet.find_fallback_nic() with cloudnet.EphemeralIPv4Network(nic, "169.254.0.1", 16, "169.254.255.255"): @@ -75,8 +81,18 @@ class DataSourceHetzner(sources.DataSource): self.metadata['public-keys'] = md.get('public-keys', None) self.vendordata_raw = md.get("vendor_data", None) + # instance-id and serial from SMBIOS should be identical + if self.metadata['instance-id'] != serial: + raise RuntimeError( + "SMBIOS serial does not match instance ID from metadata" + ) + return True + def check_instance_id(self, sys_cfg): + return sources.instance_id_matches_system_uuid( + self.get_instance_id(), 'system-serial-number') + @property def network_config(self): """Configure the networking. This needs to be done each boot, since @@ -96,8 +112,18 @@ class DataSourceHetzner(sources.DataSource): return self._network_config -def on_hetzner(): - return util.read_dmi_data('system-manufacturer') == "Hetzner" +def get_hcloud_data(): + vendor_name = util.read_dmi_data('system-manufacturer') + if vendor_name != "Hetzner": + return (False, None) + + serial = util.read_dmi_data("system-serial-number") + if serial: + LOG.debug("Running on Hetzner Cloud: serial=%s", serial) + else: + raise RuntimeError("Hetzner Cloud detected, but no serial found") + + return (True, serial) # Used to match classes to dependencies diff --git a/tests/unittests/test_datasource/test_hetzner.py b/tests/unittests/test_datasource/test_hetzner.py index d0879545..fc3649a4 100644 --- a/tests/unittests/test_datasource/test_hetzner.py +++ b/tests/unittests/test_datasource/test_hetzner.py @@ -77,10 +77,10 @@ class TestDataSourceHetzner(CiTestCase): @mock.patch('cloudinit.net.find_fallback_nic') @mock.patch('cloudinit.sources.helpers.hetzner.read_metadata') @mock.patch('cloudinit.sources.helpers.hetzner.read_userdata') - @mock.patch('cloudinit.sources.DataSourceHetzner.on_hetzner') - def test_read_data(self, m_on_hetzner, m_usermd, m_readmd, m_fallback_nic, - m_net): - m_on_hetzner.return_value = True + @mock.patch('cloudinit.sources.DataSourceHetzner.get_hcloud_data') + def test_read_data(self, m_get_hcloud_data, m_usermd, m_readmd, + m_fallback_nic, m_net): + m_get_hcloud_data.return_value = (True, METADATA.get('instance-id')) m_readmd.return_value = METADATA.copy() m_usermd.return_value = USERDATA m_fallback_nic.return_value = 'eth0' @@ -107,11 +107,12 @@ class TestDataSourceHetzner(CiTestCase): @mock.patch('cloudinit.sources.helpers.hetzner.read_metadata') @mock.patch('cloudinit.net.find_fallback_nic') - @mock.patch('cloudinit.sources.DataSourceHetzner.on_hetzner') - def test_not_on_hetzner_returns_false(self, m_on_hetzner, m_find_fallback, - m_read_md): - """If helper 'on_hetzner' returns False, return False from get_data.""" - m_on_hetzner.return_value = False + @mock.patch('cloudinit.sources.DataSourceHetzner.get_hcloud_data') + def test_not_on_hetzner_returns_false(self, m_get_hcloud_data, + m_find_fallback, m_read_md): + """If helper 'get_hcloud_data' returns False, + return False from get_data.""" + m_get_hcloud_data.return_value = (False, None) ds = self.get_ds() ret = ds.get_data() -- cgit v1.2.3 From a6afe9c574681632c3c82d6bc8c6a7fb91425bdf Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Thu, 29 Oct 2020 21:41:39 +0100 Subject: tools/build-on-freebsd: fix comment explaining purpose of the script (#635) Since there is now an official port for cloud-init in FreeBSD, this script has a different purpose. It's an interface between cloud-init maintainers, and the FreeBSD port maintainers for quickly seeing which dependencies have changed. --- tools/build-on-freebsd | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 3211c355..94b03433 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -1,7 +1,10 @@ #!/bin/sh -# Since there is no official FreeBSD port yet, we need some way of building and -# installing cloud-init. This script takes care of building and installing. It -# will optionally make a first run at the end. +# The official way to install cloud-init on FreeBSD is via the net/cloud-init +# port or the (appropriately py-flavoured) package. +# This script provides a communication medium between the cloud-init maintainers +# and the cloud-init port maintainers as to what dependencies have changed. +# It also provides a quick and dirty way of building and installing, and will +# optionally make a first run at the end. set -eux -- cgit v1.2.3 From f72d0cb7a03cc311700ea8795edf2454172c7414 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 30 Oct 2020 12:33:37 -0400 Subject: .travis.yml: use a known-working version of lxd (#643) The latest lxd 4.7 snap causes failures; while we're diagnosing those and addressing the issues (either upstream or in our code), pin at lxd 4.6 so we can continue to land PRs in cloud-init trunk. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 87841424..4342aacb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: # pylxd talks only to the lxd from snap - sudo apt remove --purge lxd lxd-client - sudo rm -Rf /var/lib/lxd - - sudo snap install lxd + - sudo snap install --channel=4.6/stable lxd - sudo lxd init --auto - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles # Move any cached lxd images into lxd's image dir -- cgit v1.2.3 From 0f8be879073148f1d67094df9ec895a873caa0d7 Mon Sep 17 00:00:00 2001 From: Markus Schade Date: Fri, 30 Oct 2020 19:12:25 +0100 Subject: Hetzner: Fix instance_id / SMBIOS serial comparison (#640) Fixes erroneous string/int comparison introduced in 1431c8a metadata['instance-id'] is an integer but the value read from smbios is a string. The comparision would cause TypeError. --- cloudinit/sources/DataSourceHetzner.py | 2 +- tests/unittests/test_datasource/test_hetzner.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 1d965bf7..8e4d4b69 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -82,7 +82,7 @@ class DataSourceHetzner(sources.DataSource): self.vendordata_raw = md.get("vendor_data", None) # instance-id and serial from SMBIOS should be identical - if self.metadata['instance-id'] != serial: + if self.get_instance_id() != serial: raise RuntimeError( "SMBIOS serial does not match instance ID from metadata" ) diff --git a/tests/unittests/test_datasource/test_hetzner.py b/tests/unittests/test_datasource/test_hetzner.py index fc3649a4..eadb92f1 100644 --- a/tests/unittests/test_datasource/test_hetzner.py +++ b/tests/unittests/test_datasource/test_hetzner.py @@ -80,7 +80,8 @@ class TestDataSourceHetzner(CiTestCase): @mock.patch('cloudinit.sources.DataSourceHetzner.get_hcloud_data') def test_read_data(self, m_get_hcloud_data, m_usermd, m_readmd, m_fallback_nic, m_net): - m_get_hcloud_data.return_value = (True, METADATA.get('instance-id')) + m_get_hcloud_data.return_value = (True, + str(METADATA.get('instance-id'))) m_readmd.return_value = METADATA.copy() m_usermd.return_value = USERDATA m_fallback_nic.return_value = 'eth0' -- cgit v1.2.3 From f8c84aeead77b7e508644d94889ee701f20e8d31 Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Fri, 30 Oct 2020 20:12:38 +0000 Subject: Correct documentation and testcase data for some user-data YAML (#618) For cc_users_groups the user setting "expiredate" must be quoted in order for the relevant flag and value to be then passed to the useradd command. It its vaiue is not quoted then it is treated as Python type datetime.date and in `cloudinit/distros/__init__.py` the below "is it a string" condition fails and so no "--expiredate" parameter is passed to useradd and therefore it has no effect: ``` if key in useradd_opts and val and isinstance(val, str): useradd_cmd.extend([useradd_opts[key], val]) ``` For cc_users_groups, the user setting "inactive" does not actually disable accounts, the useradd "--inactive" option actually defines the number of days after password expiry that users can still login. So I have changed the docs to show it taking a quoted value of days (which works with the current code) rather than a boolean value. The quotes are necessary, like expiredate above, so that the value is also passed to the useradd command. For cc_power_state_change.py the "delay" setting value needs to have quotes around it as otherwise its leading plus sign will be stripped off. --- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_users_groups.py | 9 +++++---- doc/examples/cloud-config-power-state.txt | 2 +- doc/examples/cloud-config-user-groups.txt | 7 ++++--- doc/examples/cloud-config.txt | 4 ++-- tests/cloud_tests/testcases/examples/including_user_groups.yaml | 4 ++-- tests/cloud_tests/testcases/modules/user_groups.yaml | 4 ++-- tests/data/merge_sources/expected10.yaml | 2 +- tests/data/merge_sources/expected7.yaml | 6 +++--- tests/data/merge_sources/source10-1.yaml | 2 +- tests/data/merge_sources/source7-1.yaml | 4 ++-- tests/data/merge_sources/source7-2.yaml | 2 +- 12 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index b0cfafcd..5780a7e9 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,7 +22,7 @@ The ``delay`` key specifies a duration to be added onto any shutdown command used. Therefore, if a 5 minute delay and a 120 second shutdown are specified, the maximum amount of time between cloud-init starting and the system shutting down is 7 minutes, and the minimum amount of time is 5 minutes. The ``delay`` -key must have an argument in either the form ``+5`` for 5 minutes or ``now`` +key must have an argument in either the form ``'+5'`` for 5 minutes or ``now`` for immediate shutdown. Optionally, a command can be run to determine whether or not diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index d4e923ef..ac4a4410 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -26,13 +26,14 @@ entry of the ``users`` list. Each entry in the ``users`` list, other than a config keys for an entry in ``users`` are as follows: - ``name``: The user's login name - - ``expiredate``: Optional. Date on which the user's login will be + - ``expiredate``: Optional. Date on which the user's account will be disabled. Default: none - ``gecos``: Optional. Comment about the user, usually a comma-separated string of real name and contact information. Default: none - ``groups``: Optional. Additional groups to add the user to. Default: none - ``homedir``: Optional. Home dir for user. Default is ``/home/`` - - ``inactive``: Optional. Mark user inactive. Default: false + - ``inactive``: Optional. Number of days after a password expires until + the account is permanently disabled. Default: none - ``lock_passwd``: Optional. Disable password login. Default: true - ``no_create_home``: Optional. Do not create home directory. Default: false @@ -102,11 +103,11 @@ config keys for an entry in ``users`` are as follows: - name: sudo: false - name: - expiredate: + expiredate: '' gecos: groups: homedir: - inactive: + inactive: '' lock_passwd: no_create_home: no_log_init: diff --git a/doc/examples/cloud-config-power-state.txt b/doc/examples/cloud-config-power-state.txt index 9cd56814..002707ec 100644 --- a/doc/examples/cloud-config-power-state.txt +++ b/doc/examples/cloud-config-power-state.txt @@ -18,7 +18,7 @@ # when 'timeout' seconds have elapsed. # # delay: form accepted by shutdown. default is 'now'. other format -# accepted is +m (m in minutes) +# accepted is '+m' (m in minutes) # mode: required. must be one of 'poweroff', 'halt', 'reboot' # message: provided as the message argument to 'shutdown'. default is none. # timeout: the amount of time to give the cloud-init process to finish diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index b593cdd1..4a5a7e20 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -19,7 +19,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -34,7 +34,7 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true - name: fizzbuzz sudo: False @@ -47,6 +47,7 @@ users: # Valid Values: # name: The user's login name +# expiredate: Date on which the user's account will be disabled. # gecos: The user name's real name, i.e. "Bob B. Smith" # homedir: Optional. Set to the local path you want to use. Defaults to # /home/ @@ -57,7 +58,7 @@ users: # "staff_u". When this is omitted the system will select the default # SELinux user. # lock_passwd: Defaults to true. Lock the password to disable password login -# inactive: Create the user as inactive +# inactive: Number of days after password expires until account is disabled # passwd: The hash -- not the password itself -- of the password you want # to use for this user. You can generate a safe hash via: # mkpasswd --method=SHA-512 --rounds=4096 diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index f3ae5e68..de9a0f87 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -518,10 +518,10 @@ manual_cache_clean: False # syslog being taken down while cloud-init is running. # # delay: form accepted by shutdown. default is 'now'. other format -# accepted is +m (m in minutes) +# accepted is '+m' (m in minutes) # mode: required. must be one of 'poweroff', 'halt', 'reboot' # message: provided as the message argument to 'shutdown'. default is none. power_state: - delay: 30 + delay: '+30' mode: poweroff message: Bye Bye diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml index 77528d98..86e392dd 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.yaml +++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml @@ -18,7 +18,7 @@ cloud_config: | gecos: Foo B. Bar primary_group: foobar groups: users - expiredate: 2038-01-19 + expiredate: '2038-01-19' lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo @@ -28,7 +28,7 @@ cloud_config: | lock_passwd: true - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true collect_scripts: group_ubuntu: | diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml index 675dfb8c..91b0e281 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.yaml +++ b/tests/cloud_tests/testcases/modules/user_groups.yaml @@ -17,7 +17,7 @@ cloud_config: | gecos: Foo B. Bar primary_group: foobar groups: users - expiredate: 2038-01-19 + expiredate: '2038-01-19' lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo @@ -27,7 +27,7 @@ cloud_config: | lock_passwd: true - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true collect_scripts: group_ubuntu: | diff --git a/tests/data/merge_sources/expected10.yaml b/tests/data/merge_sources/expected10.yaml index b865db16..e9f88f7b 100644 --- a/tests/data/merge_sources/expected10.yaml +++ b/tests/data/merge_sources/expected10.yaml @@ -1,7 +1,7 @@ #cloud-config power_state: - delay: 30 + delay: '+30' mode: poweroff message: [Bye, Bye, Pew, Pew] diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml index d32988e8..8186d13a 100644 --- a/tests/data/merge_sources/expected7.yaml +++ b/tests/data/merge_sources/expected7.yaml @@ -7,7 +7,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -22,7 +22,7 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true - bob - joe @@ -32,7 +32,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ diff --git a/tests/data/merge_sources/source10-1.yaml b/tests/data/merge_sources/source10-1.yaml index 6ae72a13..36fd336d 100644 --- a/tests/data/merge_sources/source10-1.yaml +++ b/tests/data/merge_sources/source10-1.yaml @@ -1,6 +1,6 @@ #cloud-config power_state: - delay: 30 + delay: '+30' mode: poweroff message: [Bye, Bye] diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml index 6405fc9b..ec93079f 100644 --- a/tests/data/merge_sources/source7-1.yaml +++ b/tests/data/merge_sources/source7-1.yaml @@ -7,7 +7,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -22,6 +22,6 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml index 0cd28978..0c02abff 100644 --- a/tests/data/merge_sources/source7-2.yaml +++ b/tests/data/merge_sources/source7-2.yaml @@ -9,7 +9,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ -- cgit v1.2.3 From 8642e8bce3530d2deb6b02895c08edd291eea48f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 2 Nov 2020 07:20:30 -0700 Subject: doc: add example query commands to debug Jinja templates (#645) --- doc/rtd/topics/instancedata.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 255245a4..1850982c 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -592,6 +592,22 @@ see only redacted values. % cloud-init query --format 'cloud: {{ v1.cloud_name }} myregion: {{ % v1.region }}' + # Locally test that your template userdata provided to the vm was rendered as + # intended. + % cloud-init query --format "$(sudo cloud-init query userdata)" + + # The --format command renders jinja templates, this can also be used + # to develop and test jinja template constructs + % cat > test-templating.yaml < Date: Mon, 2 Nov 2020 10:20:02 -0500 Subject: conftest: improve docstring for disable_subp_usage (#644) Specifically by fixing references to `util`, adding some missing formatting, and adding information about a corner case of mock usage that `disable_subp_usage` breaks. --- conftest.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index 459b708a..9e9d9ff8 100644 --- a/conftest.py +++ b/conftest.py @@ -70,20 +70,30 @@ def disable_subp_usage(request, fixture_utils): """ Across all (pytest) tests, ensure that subp.subp is not invoked. - Note that this can only catch invocations where the util module is imported - and ``subp.subp(...)`` is called. ``from cloudinit.subp mport subp`` - imports happen before the patching here (or the CiTestCase monkey-patching) - happens, so are left untouched. - - To allow a particular test method or class to use subp.subp you can mark it - as such:: + Note that this can only catch invocations where the ``subp`` module is + imported and ``subp.subp(...)`` is called. ``from cloudinit.subp import + subp`` imports happen before the patching here (or the CiTestCase + monkey-patching) happens, so are left untouched. + + While ``disable_subp_usage`` unconditionally patches + ``cloudinit.subp.subp``, any test-local patching will override this + patching (i.e. the mock created for that patch call will replace the mock + created by ``disable_subp_usage``), allowing tests to be written normally. + One important exception: if ``autospec=True`` is passed to such an + overriding patch call it will fail: autospeccing introspects the object + being patched and as ``subp.subp`` will always be a mock when that + autospeccing happens, the introspection fails. (The specific error is: + ``TypeError: name must be a str, not a MagicMock``.) + + To allow a particular test method or class to use ``subp.subp`` you can + mark it as such:: @pytest.mark.allow_all_subp def test_whoami(self): subp.subp(["whoami"]) - To instead allow subp.subp usage for a specific command, you can use the - ``allow_subp_for`` mark:: + To instead allow ``subp.subp`` usage for a specific command, you can use + the ``allow_subp_for`` mark:: @pytest.mark.allow_subp_for("bash") def test_bash(self): @@ -97,9 +107,9 @@ def disable_subp_usage(request, fixture_utils): subp.subp(["whoami"]) This fixture (roughly) mirrors the functionality of - CiTestCase.allowed_subp. N.B. While autouse fixtures do affect non-pytest - tests, CiTestCase's allowed_subp does take precedence (and we have - TestDisableSubpUsageInTestSubclass to confirm that). + ``CiTestCase.allowed_subp``. N.B. While autouse fixtures do affect + non-pytest tests, CiTestCase's ``allowed_subp`` does take precedence (and + we have ``TestDisableSubpUsageInTestSubclass`` to confirm that). """ allow_subp_for = fixture_utils.closest_marker_args_or( request, "allow_subp_for", None -- cgit v1.2.3 From 34f8e2213c42106b3e1568b5c5aac5565df954e3 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Mon, 2 Nov 2020 18:01:41 +0100 Subject: util: fix mounting of vfat on *BSD (#637) Fix mounting of vfat filesystems by normalizing the different names for vfat to "msdos" which works across BSDs. --- cloudinit/tests/test_util.py | 35 +++++++++++++++++++++++++++++++++++ cloudinit/util.py | 14 ++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 77714928..b7a302f1 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -730,6 +730,41 @@ class TestMountCb: """already_mounted_device_and_mountdict, but return only the device""" return already_mounted_device_and_mountdict[0] + @pytest.mark.parametrize( + "mtype,expected", + [ + # While the filesystem is called iso9660, the mount type is cd9660 + ("iso9660", "cd9660"), + # vfat is generally called "msdos" on BSD + ("vfat", "msdos"), + # judging from man pages, only FreeBSD has this alias + ("msdosfs", "msdos"), + # Test happy path + ("ufs", "ufs") + ], + ) + @mock.patch("cloudinit.util.is_Linux", autospec=True) + @mock.patch("cloudinit.util.is_BSD", autospec=True) + @mock.patch("cloudinit.util.subp.subp") + @mock.patch("cloudinit.temp_utils.tempdir", autospec=True) + def test_normalize_mtype_on_bsd( + self, m_tmpdir, m_subp, m_is_BSD, m_is_Linux, mtype, expected + ): + m_is_BSD.return_value = True + m_is_Linux.return_value = False + m_tmpdir.return_value.__enter__ = mock.Mock( + autospec=True, return_value="/tmp/fake" + ) + m_tmpdir.return_value.__exit__ = mock.Mock( + autospec=True, return_value=True + ) + callback = mock.Mock(autospec=True) + + util.mount_cb('/dev/fake0', callback, mtype=mtype) + assert mock.call( + ["mount", "-o", "ro", "-t", expected, "/dev/fake0", "/tmp/fake"], + update_env=None) in m_subp.call_args_list + @pytest.mark.parametrize("invalid_mtype", [int(0), float(0.0), dict()]) def test_typeerror_raised_for_invalid_mtype(self, invalid_mtype): with pytest.raises(TypeError): diff --git a/cloudinit/util.py b/cloudinit/util.py index 83727544..b8856af1 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -417,6 +417,11 @@ def multi_log(text, console=True, stderr=True, log.log(log_level, text) +@lru_cache() +def is_Linux(): + return 'Linux' in platform.system() + + @lru_cache() def is_BSD(): return 'BSD' in platform.system() @@ -1661,16 +1666,17 @@ def mount_cb(device, callback, data=None, mtype=None, _type=type(mtype))) # clean up 'mtype' input a bit based on platform. - platsys = platform.system().lower() - if platsys == "linux": + if is_Linux(): if mtypes is None: mtypes = ["auto"] - elif platsys.endswith("bsd"): + elif is_BSD(): if mtypes is None: - mtypes = ['ufs', 'cd9660', 'vfat'] + mtypes = ['ufs', 'cd9660', 'msdos'] for index, mtype in enumerate(mtypes): if mtype == "iso9660": mtypes[index] = "cd9660" + if mtype in ["vfat", "msdosfs"]: + mtypes[index] = "msdos" else: # we cannot do a smart "auto", so just call 'mount' once with no -t mtypes = [''] -- cgit v1.2.3 From 0ff34cfb94b3c480b6a697e4ad1878fa893a3a39 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 2 Nov 2020 11:39:00 -0600 Subject: Remove Ubuntu restriction from PR template (#648) --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 37f55d26..4314b9da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,7 +6,7 @@ ## Test Steps -- cgit v1.2.3 From 2ea3121fece7dcdb0d5c424cb6fc597699c0e895 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 2 Nov 2020 13:08:53 -0500 Subject: test_lp1886531: don't assume /etc/fstab exists (#639) As the bug manifested because groovy doesn't ship /etc/fstab, we should not assume that /etc/fstab will be present for us to remove in our bootcmd. Co-authored-by: Rick Harding --- tests/integration_tests/bugs/test_lp1886531.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/bugs/test_lp1886531.py b/tests/integration_tests/bugs/test_lp1886531.py index b1112185..058ea8bb 100644 --- a/tests/integration_tests/bugs/test_lp1886531.py +++ b/tests/integration_tests/bugs/test_lp1886531.py @@ -15,7 +15,7 @@ import pytest USER_DATA = """\ #cloud-config bootcmd: -- rm /etc/fstab +- rm -f /etc/fstab """ -- cgit v1.2.3 From d619f5171ac0ce5b626ef4575ad5f4468e94c987 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 2 Nov 2020 13:40:10 -0500 Subject: integration_tests: various launch improvements (#638) * integration_tests: fix passing launch_kwargs to session_cloud.launch * integration_tests: log the launch_kwargs before launching instances * integration_tests: add support for specifying instance name for tests Co-authored-by: Rick Harding --- tests/integration_tests/clouds.py | 11 ++++++++--- tests/integration_tests/conftest.py | 10 +++++++++- tox.ini | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 06f1c623..08c86198 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -54,13 +54,18 @@ class IntegrationCloud(ABC): self.settings.EXISTING_INSTANCE_ID ) return - launch_kwargs = { + kwargs = { 'image_id': self.image_id, 'user_data': user_data, 'wait': False, } - launch_kwargs.update(launch_kwargs) - pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) + kwargs.update(launch_kwargs) + log.info( + "Launching instance with launch_kwargs:\n{}".format( + "\n".join("{}={}".format(*item) for item in kwargs.items()) + ) + ) + pycloudlib_instance = self.cloud_instance.launch(**kwargs) pycloudlib_instance.wait(raise_on_cloudinit_failure=False) log.info('Launched instance: %s', pycloudlib_instance) return self.get_instance(pycloudlib_instance, settings) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 9163ac66..34e674e9 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -111,7 +111,15 @@ def _client(request, fixture_utils, session_cloud): """ user_data = fixture_utils.closest_marker_first_arg_or( request, 'user_data', None) - with session_cloud.launch(user_data=user_data) as instance: + name = fixture_utils.closest_marker_first_arg_or( + request, 'instance_name', None + ) + launch_kwargs = {} + if name is not None: + launch_kwargs = {"name": name} + with session_cloud.launch( + user_data=user_data, launch_kwargs=launch_kwargs + ) as instance: yield instance diff --git a/tox.ini b/tox.ini index a8681197..816e6e8e 100644 --- a/tox.ini +++ b/tox.ini @@ -158,3 +158,4 @@ markers = oci: test will only run on OCI platform lxd_container: test will only run in LXD container user_data: the user data to be passed to the test instance + instance_name: the name to be used for the test instance -- cgit v1.2.3 From 0af1ff1eaf593c325b4f53181a572110eb016c50 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 2 Nov 2020 15:41:11 -0500 Subject: cloudinit: move dmi functions out of util (#622) This just separates the reading of dmi values into its own file. Some things of note: * left import of util in dmi.py only for 'is_container' It'd be good if is_container was not in util. * just the use of 'util.is_x86' to dmi.py * open() is used directly rather than load_file. --- cloudinit/dmi.py | 131 ++++++++++++++++++++++ cloudinit/sources/DataSourceAliYun.py | 4 +- cloudinit/sources/DataSourceAltCloud.py | 3 +- cloudinit/sources/DataSourceAzure.py | 5 +- cloudinit/sources/DataSourceCloudSigma.py | 4 +- cloudinit/sources/DataSourceEc2.py | 9 +- cloudinit/sources/DataSourceExoscale.py | 3 +- cloudinit/sources/DataSourceGCE.py | 5 +- cloudinit/sources/DataSourceHetzner.py | 5 +- cloudinit/sources/DataSourceNoCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 5 +- cloudinit/sources/DataSourceOpenStack.py | 5 +- cloudinit/sources/DataSourceOracle.py | 5 +- cloudinit/sources/DataSourceScaleway.py | 3 +- cloudinit/sources/DataSourceSmartOS.py | 3 +- cloudinit/sources/__init__.py | 3 +- cloudinit/sources/helpers/digitalocean.py | 5 +- cloudinit/sources/tests/test_oracle.py | 6 +- cloudinit/tests/test_dmi.py | 131 ++++++++++++++++++++++ cloudinit/util.py | 119 -------------------- tests/unittests/test_datasource/test_aliyun.py | 6 +- tests/unittests/test_datasource/test_altcloud.py | 21 ++-- tests/unittests/test_datasource/test_azure.py | 10 +- tests/unittests/test_datasource/test_nocloud.py | 3 +- tests/unittests/test_datasource/test_openstack.py | 16 +-- tests/unittests/test_datasource/test_ovf.py | 16 +-- tests/unittests/test_datasource/test_scaleway.py | 8 +- tests/unittests/test_util.py | 123 -------------------- 28 files changed, 348 insertions(+), 312 deletions(-) create mode 100644 cloudinit/dmi.py create mode 100644 cloudinit/tests/test_dmi.py diff --git a/cloudinit/dmi.py b/cloudinit/dmi.py new file mode 100644 index 00000000..96e0e423 --- /dev/null +++ b/cloudinit/dmi.py @@ -0,0 +1,131 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import log as logging +from cloudinit import subp +from cloudinit.util import is_container + +import os + +LOG = logging.getLogger(__name__) + +# Path for DMI Data +DMI_SYS_PATH = "/sys/class/dmi/id" + +# dmidecode and /sys/class/dmi/id/* use different names for the same value, +# this allows us to refer to them by one canonical name +DMIDECODE_TO_DMI_SYS_MAPPING = { + 'baseboard-asset-tag': 'board_asset_tag', + 'baseboard-manufacturer': 'board_vendor', + 'baseboard-product-name': 'board_name', + 'baseboard-serial-number': 'board_serial', + 'baseboard-version': 'board_version', + 'bios-release-date': 'bios_date', + 'bios-vendor': 'bios_vendor', + 'bios-version': 'bios_version', + 'chassis-asset-tag': 'chassis_asset_tag', + 'chassis-manufacturer': 'chassis_vendor', + 'chassis-serial-number': 'chassis_serial', + 'chassis-version': 'chassis_version', + 'system-manufacturer': 'sys_vendor', + 'system-product-name': 'product_name', + 'system-serial-number': 'product_serial', + 'system-uuid': 'product_uuid', + 'system-version': 'product_version', +} + + +def _read_dmi_syspath(key): + """ + Reads dmi data with from /sys/class/dmi/id + """ + if key not in DMIDECODE_TO_DMI_SYS_MAPPING: + return None + mapped_key = DMIDECODE_TO_DMI_SYS_MAPPING[key] + dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, mapped_key) + + LOG.debug("querying dmi data %s", dmi_key_path) + if not os.path.exists(dmi_key_path): + LOG.debug("did not find %s", dmi_key_path) + return None + + try: + with open(dmi_key_path, "rb") as fp: + key_data = fp.read() + except PermissionError: + LOG.debug("Could not read %s", dmi_key_path) + return None + + # uninitialized dmi values show as all \xff and /sys appends a '\n'. + # in that event, return empty string. + if key_data == b'\xff' * (len(key_data) - 1) + b'\n': + key_data = b"" + + try: + return key_data.decode('utf8').strip() + except UnicodeDecodeError as e: + LOG.error("utf-8 decode of content (%s) in %s failed: %s", + dmi_key_path, key_data, e) + + return None + + +def _call_dmidecode(key, dmidecode_path): + """ + Calls out to dmidecode to get the data out. This is mostly for supporting + OS's without /sys/class/dmi/id support. + """ + try: + cmd = [dmidecode_path, "--string", key] + (result, _err) = subp.subp(cmd) + result = result.strip() + LOG.debug("dmidecode returned '%s' for '%s'", result, key) + if result.replace(".", "") == "": + return "" + return result + except (IOError, OSError) as e: + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, e) + return None + + +def read_dmi_data(key): + """ + Wrapper for reading DMI data. + + If running in a container return None. This is because DMI data is + assumed to be not useful in a container as it does not represent the + container but rather the host. + + This will do the following (returning the first that produces a + result): + 1) Use a mapping to translate `key` from dmidecode naming to + sysfs naming and look in /sys/class/dmi/... for a value. + 2) Use `key` as a sysfs key directly and look in /sys/class/dmi/... + 3) Fall-back to passing `key` to `dmidecode --string`. + + If all of the above fail to find a value, None will be returned. + """ + + if is_container(): + return None + + syspath_value = _read_dmi_syspath(key) + if syspath_value is not None: + return syspath_value + + def is_x86(arch): + return (arch == 'x86_64' or (arch[0] == 'i' and arch[2:] == '86')) + + # running dmidecode can be problematic on some arches (LP: #1243287) + uname_arch = os.uname()[4] + if not (is_x86(uname_arch) or uname_arch in ('aarch64', 'amd64')): + LOG.debug("dmidata is not supported on %s", uname_arch) + return None + + dmidecode_path = subp.which('dmidecode') + if dmidecode_path: + return _call_dmidecode(key, dmidecode_path) + + LOG.warning("did not find either path %s or dmidecode command", + DMI_SYS_PATH) + return None + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 45cc9f00..09052873 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -1,8 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import dmi from cloudinit import sources from cloudinit.sources import DataSourceEc2 as EC2 -from cloudinit import util ALIYUN_PRODUCT = "Alibaba Cloud ECS" @@ -30,7 +30,7 @@ class DataSourceAliYun(EC2.DataSourceEc2): def _is_aliyun(): - return util.read_dmi_data('system-product-name') == ALIYUN_PRODUCT + return dmi.read_dmi_data('system-product-name') == ALIYUN_PRODUCT def parse_public_keys(public_keys): diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index ac3ecc3d..cd93412a 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -16,6 +16,7 @@ import errno import os import os.path +from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources from cloudinit import subp @@ -109,7 +110,7 @@ class DataSourceAltCloud(sources.DataSource): CLOUD_INFO_FILE) return 'UNKNOWN' return cloud_type - system_name = util.read_dmi_data("system-product-name") + system_name = dmi.read_dmi_data("system-product-name") if not system_name: return 'UNKNOWN' diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 70e32f46..fa3e0a2b 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -15,6 +15,7 @@ from time import time from xml.dom import minidom import xml.etree.ElementTree as ET +from cloudinit import dmi from cloudinit import log as logging from cloudinit import net from cloudinit.event import EventType @@ -630,7 +631,7 @@ class DataSourceAzure(sources.DataSource): def _iid(self, previous=None): prev_iid_path = os.path.join( self.paths.get_cpath('data'), 'instance-id') - iid = util.read_dmi_data('system-uuid') + iid = dmi.read_dmi_data('system-uuid') if os.path.exists(prev_iid_path): previous = util.load_file(prev_iid_path).strip() if is_byte_swapped(previous, iid): @@ -1630,7 +1631,7 @@ def _is_platform_viable(seed_dir): description="found azure asset tag", parent=azure_ds_reporter ) as evt: - asset_tag = util.read_dmi_data('chassis-asset-tag') + asset_tag = dmi.read_dmi_data('chassis-asset-tag') if asset_tag == AZURE_CHASSIS_ASSET_TAG: return True msg = "Non-Azure DMI asset tag '%s' discovered." % asset_tag diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index df88f677..f63baf74 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -9,9 +9,9 @@ import re from cloudinit.cs_utils import Cepko, SERIAL_PORT +from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources -from cloudinit import util LOG = logging.getLogger(__name__) @@ -38,7 +38,7 @@ class DataSourceCloudSigma(sources.DataSource): """ LOG.debug("determining hypervisor product name via dmi data") - sys_product_name = util.read_dmi_data("system-product-name") + sys_product_name = dmi.read_dmi_data("system-product-name") if not sys_product_name: LOG.debug("system-product-name not available in dmi data") return False diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 1d09c12a..1930a509 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -11,6 +11,7 @@ import os import time +from cloudinit import dmi from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import net @@ -699,26 +700,26 @@ def _collect_platform_data(): uuid = util.load_file("/sys/hypervisor/uuid").strip() data['uuid_source'] = 'hypervisor' except Exception: - uuid = util.read_dmi_data('system-uuid') + uuid = dmi.read_dmi_data('system-uuid') data['uuid_source'] = 'dmi' if uuid is None: uuid = '' data['uuid'] = uuid.lower() - serial = util.read_dmi_data('system-serial-number') + serial = dmi.read_dmi_data('system-serial-number') if serial is None: serial = '' data['serial'] = serial.lower() - asset_tag = util.read_dmi_data('chassis-asset-tag') + asset_tag = dmi.read_dmi_data('chassis-asset-tag') if asset_tag is None: asset_tag = '' data['asset_tag'] = asset_tag.lower() - vendor = util.read_dmi_data('system-manufacturer') + vendor = dmi.read_dmi_data('system-manufacturer') data['vendor'] = (vendor if vendor else '').lower() return data diff --git a/cloudinit/sources/DataSourceExoscale.py b/cloudinit/sources/DataSourceExoscale.py index d59aefd1..adee6d79 100644 --- a/cloudinit/sources/DataSourceExoscale.py +++ b/cloudinit/sources/DataSourceExoscale.py @@ -3,6 +3,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import dmi from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources @@ -135,7 +136,7 @@ class DataSourceExoscale(sources.DataSource): return self.extra_config def _is_platform_viable(self): - return util.read_dmi_data('system-product-name').startswith( + return dmi.read_dmi_data('system-product-name').startswith( EXOSCALE_DMI_NAME) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 0ec5f6ec..746caddb 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -7,6 +7,7 @@ import json from base64 import b64decode +from cloudinit import dmi from cloudinit.distros import ug_util from cloudinit import log as logging from cloudinit import sources @@ -248,12 +249,12 @@ def read_md(address=None, platform_check=True): def platform_reports_gce(): - pname = util.read_dmi_data('system-product-name') or "N/A" + pname = dmi.read_dmi_data('system-product-name') or "N/A" if pname == "Google Compute Engine": return True # system-product-name is not always guaranteed (LP: #1674861) - serial = util.read_dmi_data('system-serial-number') or "N/A" + serial = dmi.read_dmi_data('system-serial-number') or "N/A" if serial.startswith("GoogleCloud-"): return True diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 8e4d4b69..c7c88dd7 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -6,6 +6,7 @@ """Hetzner Cloud API Documentation https://docs.hetzner.cloud/""" +from cloudinit import dmi from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import sources @@ -113,11 +114,11 @@ class DataSourceHetzner(sources.DataSource): def get_hcloud_data(): - vendor_name = util.read_dmi_data('system-manufacturer') + vendor_name = dmi.read_dmi_data('system-manufacturer') if vendor_name != "Hetzner": return (False, None) - serial = util.read_dmi_data("system-serial-number") + serial = dmi.read_dmi_data("system-serial-number") if serial: LOG.debug("Running on Hetzner Cloud: serial=%s", serial) else: diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index d4a175e8..a126aad3 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -11,6 +11,7 @@ import errno import os +from cloudinit import dmi from cloudinit import log as logging from cloudinit.net import eni from cloudinit import sources @@ -61,7 +62,7 @@ class DataSourceNoCloud(sources.DataSource): # Parse the system serial label from dmi. If not empty, try parsing # like the commandline md = {} - serial = util.read_dmi_data('system-serial-number') + serial = dmi.read_dmi_data('system-serial-number') if serial and load_cmdline_data(md, serial): found.append("dmi") mydata = _merge_new_seed(mydata, {'meta-data': md}) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index a5ccb8f6..741c140a 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -14,6 +14,7 @@ import re import time from xml.dom import minidom +from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources from cloudinit import subp @@ -83,7 +84,7 @@ class DataSourceOVF(sources.DataSource): (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - system_type = util.read_dmi_data("system-product-name") + system_type = dmi.read_dmi_data("system-product-name") if system_type is None: LOG.debug("No system-product-name found") @@ -322,7 +323,7 @@ class DataSourceOVF(sources.DataSource): return True def _get_subplatform(self): - system_type = util.read_dmi_data("system-product-name").lower() + system_type = dmi.read_dmi_data("system-product-name").lower() if system_type == 'vmware': return 'vmware (%s)' % self.seed return 'ovf (%s)' % self.seed diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0ede0a0e..b3406c67 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -6,6 +6,7 @@ import time +from cloudinit import dmi from cloudinit import log as logging from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError from cloudinit import sources @@ -225,10 +226,10 @@ def detect_openstack(accept_oracle=False): """Return True when a potential OpenStack platform is detected.""" if not util.is_x86(): return True # Non-Intel cpus don't properly report dmi product names - product_name = util.read_dmi_data('system-product-name') + product_name = dmi.read_dmi_data('system-product-name') if product_name in VALID_DMI_PRODUCT_NAMES: return True - elif util.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS: + elif dmi.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS: return True elif accept_oracle and oracle._is_platform_viable(): return True diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py index 20d6487d..bf81b10b 100644 --- a/cloudinit/sources/DataSourceOracle.py +++ b/cloudinit/sources/DataSourceOracle.py @@ -17,6 +17,7 @@ import base64 from collections import namedtuple from contextlib import suppress as noop +from cloudinit import dmi from cloudinit import log as logging from cloudinit import net, sources, util from cloudinit.net import ( @@ -273,12 +274,12 @@ class DataSourceOracle(sources.DataSource): def _read_system_uuid(): - sys_uuid = util.read_dmi_data('system-uuid') + sys_uuid = dmi.read_dmi_data('system-uuid') return None if sys_uuid is None else sys_uuid.lower() def _is_platform_viable(): - asset_tag = util.read_dmi_data('chassis-asset-tag') + asset_tag = dmi.read_dmi_data('chassis-asset-tag') return asset_tag == CHASSIS_ASSET_TAG diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py index 83c2bf65..41be7665 100644 --- a/cloudinit/sources/DataSourceScaleway.py +++ b/cloudinit/sources/DataSourceScaleway.py @@ -25,6 +25,7 @@ import requests from requests.packages.urllib3.connection import HTTPConnection from requests.packages.urllib3.poolmanager import PoolManager +from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper @@ -56,7 +57,7 @@ def on_scaleway(): * the initrd created the file /var/run/scaleway. * "scaleway" is in the kernel cmdline. """ - vendor_name = util.read_dmi_data('system-manufacturer') + vendor_name = dmi.read_dmi_data('system-manufacturer') if vendor_name == 'Scaleway': return True diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f1f903bc..fd292baa 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -30,6 +30,7 @@ import random import re import socket +from cloudinit import dmi from cloudinit import log as logging from cloudinit import serial from cloudinit import sources @@ -767,7 +768,7 @@ def get_smartos_environ(uname_version=None, product_name=None): return SMARTOS_ENV_LX_BRAND if product_name is None: - system_type = util.read_dmi_data("system-product-name") + system_type = dmi.read_dmi_data("system-product-name") else: system_type = product_name diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index c4d60fff..9dccc687 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -14,6 +14,7 @@ import json import os from collections import namedtuple +from cloudinit import dmi from cloudinit import importer from cloudinit import log as logging from cloudinit import net @@ -809,7 +810,7 @@ def instance_id_matches_system_uuid(instance_id, field='system-uuid'): if not instance_id: return False - dmi_value = util.read_dmi_data(field) + dmi_value = dmi.read_dmi_data(field) if not dmi_value: return False return instance_id.lower() == dmi_value.lower() diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py index b545c4d6..f9be4ecb 100644 --- a/cloudinit/sources/helpers/digitalocean.py +++ b/cloudinit/sources/helpers/digitalocean.py @@ -5,6 +5,7 @@ import json import random +from cloudinit import dmi from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import url_helper @@ -195,11 +196,11 @@ def read_sysinfo(): # SMBIOS information # Detect if we are on DigitalOcean and return the Droplet's ID - vendor_name = util.read_dmi_data("system-manufacturer") + vendor_name = dmi.read_dmi_data("system-manufacturer") if vendor_name != "DigitalOcean": return (False, None) - droplet_id = util.read_dmi_data("system-serial-number") + droplet_id = dmi.read_dmi_data("system-serial-number") if droplet_id: LOG.debug("system identified via SMBIOS as DigitalOcean Droplet: %s", droplet_id) diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py index 7bd23813..a7bbdfd9 100644 --- a/cloudinit/sources/tests/test_oracle.py +++ b/cloudinit/sources/tests/test_oracle.py @@ -153,20 +153,20 @@ class TestDataSourceOracle: class TestIsPlatformViable(test_helpers.CiTestCase): - @mock.patch(DS_PATH + ".util.read_dmi_data", + @mock.patch(DS_PATH + ".dmi.read_dmi_data", return_value=oracle.CHASSIS_ASSET_TAG) def test_expected_viable(self, m_read_dmi_data): """System with known chassis tag is viable.""" self.assertTrue(oracle._is_platform_viable()) m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')]) - @mock.patch(DS_PATH + ".util.read_dmi_data", return_value=None) + @mock.patch(DS_PATH + ".dmi.read_dmi_data", return_value=None) def test_expected_not_viable_dmi_data_none(self, m_read_dmi_data): """System without known chassis tag is not viable.""" self.assertFalse(oracle._is_platform_viable()) m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')]) - @mock.patch(DS_PATH + ".util.read_dmi_data", return_value="LetsGoCubs") + @mock.patch(DS_PATH + ".dmi.read_dmi_data", return_value="LetsGoCubs") def test_expected_not_viable_other(self, m_read_dmi_data): """System with unnown chassis tag is not viable.""" self.assertFalse(oracle._is_platform_viable()) diff --git a/cloudinit/tests/test_dmi.py b/cloudinit/tests/test_dmi.py new file mode 100644 index 00000000..4a8af257 --- /dev/null +++ b/cloudinit/tests/test_dmi.py @@ -0,0 +1,131 @@ +from cloudinit.tests import helpers +from cloudinit import dmi +from cloudinit import util +from cloudinit import subp + +import os +import tempfile +import shutil +from unittest import mock + + +class TestReadDMIData(helpers.FilesystemMockingTestCase): + + def setUp(self): + super(TestReadDMIData, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + self.reRoot(self.new_root) + p = mock.patch("cloudinit.dmi.is_container", return_value=False) + self.addCleanup(p.stop) + self._m_is_container = p.start() + + def _create_sysfs_parent_directory(self): + util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) + + def _create_sysfs_file(self, key, content): + """Mocks the sys path found on Linux systems.""" + self._create_sysfs_parent_directory() + dmi_key = "/sys/class/dmi/id/{0}".format(key) + util.write_file(dmi_key, content) + + def _configure_dmidecode_return(self, key, content, error=None): + """ + In order to test a missing sys path and call outs to dmidecode, this + function fakes the results of dmidecode to test the results. + """ + def _dmidecode_subp(cmd): + if cmd[-1] != key: + raise subp.ProcessExecutionError() + return (content, error) + + self.patched_funcs.enter_context( + mock.patch("cloudinit.dmi.subp.which", side_effect=lambda _: True)) + self.patched_funcs.enter_context( + mock.patch("cloudinit.dmi.subp.subp", side_effect=_dmidecode_subp)) + + def patch_mapping(self, new_mapping): + self.patched_funcs.enter_context( + mock.patch('cloudinit.dmi.DMIDECODE_TO_DMI_SYS_MAPPING', + new_mapping)) + + def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): + self.patch_mapping({'mapped-key': 'mapped-value'}) + expected_dmi_value = 'sys-used-correctly' + self._create_sysfs_file('mapped-value', expected_dmi_value) + self._configure_dmidecode_return('mapped-key', 'wrong-wrong-wrong') + self.assertEqual(expected_dmi_value, dmi.read_dmi_data('mapped-key')) + + def test_dmidecode_used_if_no_sysfs_file_on_disk(self): + self.patch_mapping({}) + self._create_sysfs_parent_directory() + expected_dmi_value = 'dmidecode-used' + self._configure_dmidecode_return('use-dmidecode', expected_dmi_value) + with mock.patch("cloudinit.util.os.uname") as m_uname: + m_uname.return_value = ('x-sysname', 'x-nodename', + 'x-release', 'x-version', 'x86_64') + self.assertEqual(expected_dmi_value, + dmi.read_dmi_data('use-dmidecode')) + + def test_dmidecode_not_used_on_arm(self): + self.patch_mapping({}) + print("current =%s", subp) + self._create_sysfs_parent_directory() + dmi_val = 'from-dmidecode' + dmi_name = 'use-dmidecode' + self._configure_dmidecode_return(dmi_name, dmi_val) + print("now =%s", subp) + + expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val} + found = {} + # we do not run the 'dmi-decode' binary on some arches + # verify that anything requested that is not in the sysfs dir + # will return None on those arches. + with mock.patch("cloudinit.util.os.uname") as m_uname: + for arch in expected: + m_uname.return_value = ('x-sysname', 'x-nodename', + 'x-release', 'x-version', arch) + print("now2 =%s", subp) + found[arch] = dmi.read_dmi_data(dmi_name) + self.assertEqual(expected, found) + + def test_none_returned_if_neither_source_has_data(self): + self.patch_mapping({}) + self._configure_dmidecode_return('key', 'value') + self.assertIsNone(dmi.read_dmi_data('expect-fail')) + + def test_none_returned_if_dmidecode_not_in_path(self): + self.patched_funcs.enter_context( + mock.patch.object(subp, 'which', lambda _: False)) + self.patch_mapping({}) + self.assertIsNone(dmi.read_dmi_data('expect-fail')) + + def test_empty_string_returned_instead_of_foxfox(self): + # uninitialized dmi values show as \xff, return empty string + my_len = 32 + dmi_value = b'\xff' * my_len + b'\n' + expected = "" + dmi_key = 'system-product-name' + sysfs_key = 'product_name' + self._create_sysfs_file(sysfs_key, dmi_value) + self.assertEqual(expected, dmi.read_dmi_data(dmi_key)) + + def test_container_returns_none(self): + """In a container read_dmi_data should always return None.""" + + # first verify we get the value if not in container + self._m_is_container.return_value = False + key, val = ("system-product-name", "my_product") + self._create_sysfs_file('product_name', val) + self.assertEqual(val, dmi.read_dmi_data(key)) + + # then verify in container returns None + self._m_is_container.return_value = True + self.assertIsNone(dmi.read_dmi_data(key)) + + def test_container_returns_none_on_unknown(self): + """In a container even bogus keys return None.""" + self._m_is_container.return_value = True + self._create_sysfs_file('product_name', "should-be-ignored") + self.assertIsNone(dmi.read_dmi_data("bogus")) + self.assertIsNone(dmi.read_dmi_data("system-product-name")) diff --git a/cloudinit/util.py b/cloudinit/util.py index b8856af1..bdb3694d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -159,32 +159,6 @@ def fully_decoded_payload(part): return cte_payload -# Path for DMI Data -DMI_SYS_PATH = "/sys/class/dmi/id" - -# dmidecode and /sys/class/dmi/id/* use different names for the same value, -# this allows us to refer to them by one canonical name -DMIDECODE_TO_DMI_SYS_MAPPING = { - 'baseboard-asset-tag': 'board_asset_tag', - 'baseboard-manufacturer': 'board_vendor', - 'baseboard-product-name': 'board_name', - 'baseboard-serial-number': 'board_serial', - 'baseboard-version': 'board_version', - 'bios-release-date': 'bios_date', - 'bios-vendor': 'bios_vendor', - 'bios-version': 'bios_version', - 'chassis-asset-tag': 'chassis_asset_tag', - 'chassis-manufacturer': 'chassis_vendor', - 'chassis-serial-number': 'chassis_serial', - 'chassis-version': 'chassis_version', - 'system-manufacturer': 'sys_vendor', - 'system-product-name': 'product_name', - 'system-serial-number': 'product_serial', - 'system-uuid': 'product_uuid', - 'system-version': 'product_version', -} - - class SeLinuxGuard(object): def __init__(self, path, recursive=False): # Late import since it might not always @@ -2421,57 +2395,6 @@ def human2bytes(size): return int(num * mpliers[mplier]) -def _read_dmi_syspath(key): - """ - Reads dmi data with from /sys/class/dmi/id - """ - if key not in DMIDECODE_TO_DMI_SYS_MAPPING: - return None - mapped_key = DMIDECODE_TO_DMI_SYS_MAPPING[key] - dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, mapped_key) - LOG.debug("querying dmi data %s", dmi_key_path) - try: - if not os.path.exists(dmi_key_path): - LOG.debug("did not find %s", dmi_key_path) - return None - - key_data = load_file(dmi_key_path, decode=False) - if not key_data: - LOG.debug("%s did not return any data", dmi_key_path) - return None - - # uninitialized dmi values show as all \xff and /sys appends a '\n'. - # in that event, return a string of '.' in the same length. - if key_data == b'\xff' * (len(key_data) - 1) + b'\n': - key_data = b"" - - str_data = key_data.decode('utf8').strip() - LOG.debug("dmi data %s returned %s", dmi_key_path, str_data) - return str_data - - except Exception: - logexc(LOG, "failed read of %s", dmi_key_path) - return None - - -def _call_dmidecode(key, dmidecode_path): - """ - Calls out to dmidecode to get the data out. This is mostly for supporting - OS's without /sys/class/dmi/id support. - """ - try: - cmd = [dmidecode_path, "--string", key] - (result, _err) = subp.subp(cmd) - result = result.strip() - LOG.debug("dmidecode returned '%s' for '%s'", result, key) - if result.replace(".", "") == "": - return "" - return result - except (IOError, OSError) as e: - LOG.debug('failed dmidecode cmd: %s\n%s', cmd, e) - return None - - def is_x86(uname_arch=None): """Return True if platform is x86-based""" if uname_arch is None: @@ -2482,48 +2405,6 @@ def is_x86(uname_arch=None): return x86_arch_match -def read_dmi_data(key): - """ - Wrapper for reading DMI data. - - If running in a container return None. This is because DMI data is - assumed to be not useful in a container as it does not represent the - container but rather the host. - - This will do the following (returning the first that produces a - result): - 1) Use a mapping to translate `key` from dmidecode naming to - sysfs naming and look in /sys/class/dmi/... for a value. - 2) Use `key` as a sysfs key directly and look in /sys/class/dmi/... - 3) Fall-back to passing `key` to `dmidecode --string`. - - If all of the above fail to find a value, None will be returned. - """ - - if is_container(): - return None - - syspath_value = _read_dmi_syspath(key) - if syspath_value is not None: - return syspath_value - - # running dmidecode can be problematic on some arches (LP: #1243287) - uname_arch = os.uname()[4] - if not (is_x86(uname_arch) or - uname_arch == 'aarch64' or - uname_arch == 'amd64'): - LOG.debug("dmidata is not supported on %s", uname_arch) - return None - - dmidecode_path = subp.which('dmidecode') - if dmidecode_path: - return _call_dmidecode(key, dmidecode_path) - - LOG.warning("did not find either path %s or dmidecode command", - DMI_SYS_PATH) - return None - - def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(io.StringIO(string)) diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index b626229e..eb2828d5 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -188,7 +188,7 @@ class TestIsAliYun(test_helpers.CiTestCase): ALIYUN_PRODUCT = 'Alibaba Cloud ECS' read_dmi_data_expected = [mock.call('system-product-name')] - @mock.patch("cloudinit.sources.DataSourceAliYun.util.read_dmi_data") + @mock.patch("cloudinit.sources.DataSourceAliYun.dmi.read_dmi_data") def test_true_on_aliyun_product(self, m_read_dmi_data): """Should return true if the dmi product data has expected value.""" m_read_dmi_data.return_value = self.ALIYUN_PRODUCT @@ -197,7 +197,7 @@ class TestIsAliYun(test_helpers.CiTestCase): m_read_dmi_data.call_args_list) self.assertEqual(True, ret) - @mock.patch("cloudinit.sources.DataSourceAliYun.util.read_dmi_data") + @mock.patch("cloudinit.sources.DataSourceAliYun.dmi.read_dmi_data") def test_false_on_empty_string(self, m_read_dmi_data): """Should return false on empty value returned.""" m_read_dmi_data.return_value = "" @@ -206,7 +206,7 @@ class TestIsAliYun(test_helpers.CiTestCase): m_read_dmi_data.call_args_list) self.assertEqual(False, ret) - @mock.patch("cloudinit.sources.DataSourceAliYun.util.read_dmi_data") + @mock.patch("cloudinit.sources.DataSourceAliYun.dmi.read_dmi_data") def test_false_on_unknown_string(self, m_read_dmi_data): """Should return false on an unrelated string.""" m_read_dmi_data.return_value = "cubs win" diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index fc59d1d5..7a5393ac 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -14,6 +14,7 @@ import os import shutil import tempfile +from cloudinit import dmi from cloudinit import helpers from cloudinit import subp from cloudinit import util @@ -88,14 +89,14 @@ class TestGetCloudType(CiTestCase): super(TestGetCloudType, self).setUp() self.tmp = self.tmp_dir() self.paths = helpers.Paths({'cloud_dir': self.tmp}) - self.dmi_data = util.read_dmi_data + self.dmi_data = dmi.read_dmi_data # We have a different code path for arm to deal with LP1243287 # We have to switch arch to x86_64 to avoid test failure force_arch('x86_64') def tearDown(self): # Reset - util.read_dmi_data = self.dmi_data + dmi.read_dmi_data = self.dmi_data force_arch() def test_cloud_info_file_ioerror(self): @@ -123,7 +124,7 @@ class TestGetCloudType(CiTestCase): Test method get_cloud_type() for RHEVm systems. Forcing read_dmi_data return to match a RHEVm system: RHEV Hypervisor ''' - util.read_dmi_data = _dmi_data('RHEV') + dmi.read_dmi_data = _dmi_data('RHEV') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual('RHEV', dsrc.get_cloud_type()) @@ -132,7 +133,7 @@ class TestGetCloudType(CiTestCase): Test method get_cloud_type() for vSphere systems. Forcing read_dmi_data return to match a vSphere system: RHEV Hypervisor ''' - util.read_dmi_data = _dmi_data('VMware Virtual Platform') + dmi.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual('VSPHERE', dsrc.get_cloud_type()) @@ -141,7 +142,7 @@ class TestGetCloudType(CiTestCase): Test method get_cloud_type() for unknown systems. Forcing read_dmi_data return to match an unrecognized return. ''' - util.read_dmi_data = _dmi_data('Unrecognized Platform') + dmi.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual('UNKNOWN', dsrc.get_cloud_type()) @@ -219,7 +220,7 @@ class TestGetDataNoCloudInfoFile(CiTestCase): self.tmp = self.tmp_dir() self.paths = helpers.Paths( {'cloud_dir': self.tmp, 'run_dir': self.tmp}) - self.dmi_data = util.read_dmi_data + self.dmi_data = dmi.read_dmi_data dsac.CLOUD_INFO_FILE = \ 'no such file' # We have a different code path for arm to deal with LP1243287 @@ -230,14 +231,14 @@ class TestGetDataNoCloudInfoFile(CiTestCase): # Reset dsac.CLOUD_INFO_FILE = \ '/etc/sysconfig/cloud-info' - util.read_dmi_data = self.dmi_data + dmi.read_dmi_data = self.dmi_data # Return back to original arch force_arch() def test_rhev_no_cloud_file(self): '''Test No cloud info file module get_data() forcing RHEV.''' - util.read_dmi_data = _dmi_data('RHEV Hypervisor') + dmi.read_dmi_data = _dmi_data('RHEV Hypervisor') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: True self.assertEqual(True, dsrc.get_data()) @@ -245,7 +246,7 @@ class TestGetDataNoCloudInfoFile(CiTestCase): def test_vsphere_no_cloud_file(self): '''Test No cloud info file module get_data() forcing VSPHERE.''' - util.read_dmi_data = _dmi_data('VMware Virtual Platform') + dmi.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: True self.assertEqual(True, dsrc.get_data()) @@ -253,7 +254,7 @@ class TestGetDataNoCloudInfoFile(CiTestCase): def test_failure_no_cloud_file(self): '''Test No cloud info file module get_data() forcing unrecognized.''' - util.read_dmi_data = _dmi_data('Unrecognized Platform') + dmi.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.get_data()) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 56c1cf18..433fbc66 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -570,7 +570,7 @@ scbus-1 on xpt0 bus 0 (dsaz, 'set_hostname', mock.MagicMock()), (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), (dsaz.subp, 'which', lambda x: True), - (dsaz.util, 'read_dmi_data', mock.MagicMock( + (dsaz.dmi, 'read_dmi_data', mock.MagicMock( side_effect=_dmi_mocks)), (dsaz.util, 'wait_for_files', mock.MagicMock( side_effect=_wait_for_files)), @@ -1427,7 +1427,7 @@ class TestAzureBounce(CiTestCase): raise RuntimeError('should not get here') self.patches.enter_context( - mock.patch.object(dsaz.util, 'read_dmi_data', + mock.patch.object(dsaz.dmi, 'read_dmi_data', mock.MagicMock(side_effect=_dmi_mocks))) def setUp(self): @@ -2294,14 +2294,14 @@ class TestWBIsPlatformViable(CiTestCase): """White box tests for _is_platform_viable.""" with_logs = True - @mock.patch(MOCKPATH + 'util.read_dmi_data') + @mock.patch(MOCKPATH + 'dmi.read_dmi_data') def test_true_on_non_azure_chassis(self, m_read_dmi_data): """Return True if DMI chassis-asset-tag is AZURE_CHASSIS_ASSET_TAG.""" m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG self.assertTrue(dsaz._is_platform_viable('doesnotmatter')) @mock.patch(MOCKPATH + 'os.path.exists') - @mock.patch(MOCKPATH + 'util.read_dmi_data') + @mock.patch(MOCKPATH + 'dmi.read_dmi_data') def test_true_on_azure_ovf_env_in_seed_dir(self, m_read_dmi_data, m_exist): """Return True if ovf-env.xml exists in known seed dirs.""" # Non-matching Azure chassis-asset-tag @@ -2322,7 +2322,7 @@ class TestWBIsPlatformViable(CiTestCase): MOCKPATH, {'os.path.exists': False, # Non-matching Azure chassis-asset-tag - 'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X', + 'dmi.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X', 'subp.which': None}, dsaz._is_platform_viable, 'doesnotmatter')) self.assertIn( diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 2e6b53ff..02cc9b38 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import dmi from cloudinit import helpers from cloudinit.sources.DataSourceNoCloud import ( DataSourceNoCloud as dsNoCloud, @@ -30,7 +31,7 @@ class TestNoCloudDataSource(CiTestCase): self.mocks.enter_context( mock.patch.object(util, 'get_cmdline', return_value=self.cmdline)) self.mocks.enter_context( - mock.patch.object(util, 'read_dmi_data', return_value=None)) + mock.patch.object(dmi, 'read_dmi_data', return_value=None)) def _test_fs_config_is_read(self, fs_label, fs_label_to_search): vfat_device = 'device-1' diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 9b0c1b8a..415755aa 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -459,7 +459,7 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == True') @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_not_detect_openstack_intel_x86_ec2(self, m_dmi, m_proc_env, m_is_x86): """Return False on EC2 platforms.""" @@ -479,7 +479,7 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == False on EC2') m_proc_env.assert_called_with(1) - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_intel_product_name_compute(self, m_dmi, m_is_x86): """Return True on OpenStack compute and nova instances.""" @@ -491,7 +491,7 @@ class TestDetectOpenStack(test_helpers.CiTestCase): self.assertTrue( ds.detect_openstack(), 'Failed to detect_openstack') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_opentelekomcloud_chassis_asset_tag(self, m_dmi, m_is_x86): """Return True on OpenStack reporting OpenTelekomCloud asset-tag.""" @@ -509,7 +509,7 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == True on OpenTelekomCloud') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_sapccloud_chassis_asset_tag(self, m_dmi, m_is_x86): """Return True on OpenStack reporting SAP CCloud VM asset-tag.""" @@ -527,7 +527,7 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == True on SAP CCloud VM') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_oraclecloud_chassis_asset_tag(self, m_dmi, m_is_x86): """Return True on OpenStack reporting Oracle cloud asset-tag.""" @@ -566,20 +566,20 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == True on Generic OpenStack Platform') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_nova_chassis_asset_tag(self, m_dmi, m_is_x86): self._test_detect_openstack_nova_compute_chassis_asset_tag( m_dmi, m_is_x86, 'OpenStack Nova') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_compute_chassis_asset_tag(self, m_dmi, m_is_x86): self._test_detect_openstack_nova_compute_chassis_asset_tag( m_dmi, m_is_x86, 'OpenStack Compute') @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env') - @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + @test_helpers.mock.patch(MOCK_PATH + 'dmi.read_dmi_data') def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env, m_is_x86): """Return True when nova product_name specified in /proc/1/environ.""" diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 1d088577..16773de5 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -129,7 +129,7 @@ class TestDatasourceOVF(CiTestCase): ds = self.datasource(sys_cfg={}, distro={}, paths=paths) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': None, + {'dmi.read_dmi_data': None, 'transport_iso9660': NOT_FOUND, 'transport_vmware_guestinfo': NOT_FOUND}, ds.get_data) @@ -145,7 +145,7 @@ class TestDatasourceOVF(CiTestCase): paths=paths) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware', + {'dmi.read_dmi_data': 'vmware', 'transport_iso9660': NOT_FOUND, 'transport_vmware_guestinfo': NOT_FOUND}, ds.get_data) @@ -174,7 +174,7 @@ class TestDatasourceOVF(CiTestCase): with self.assertRaises(CustomScriptNotFound) as context: wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware', + {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, @@ -211,7 +211,7 @@ class TestDatasourceOVF(CiTestCase): with self.assertRaises(RuntimeError) as context: wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware', + {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, @@ -246,7 +246,7 @@ class TestDatasourceOVF(CiTestCase): with self.assertRaises(CustomScriptNotFound) as context: wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware', + {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, @@ -290,7 +290,7 @@ class TestDatasourceOVF(CiTestCase): with self.assertRaises(CustomScriptNotFound) as context: wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware', + {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, @@ -313,7 +313,7 @@ class TestDatasourceOVF(CiTestCase): self.assertEqual('ovf', ds.cloud_name) self.assertEqual('ovf', ds.platform_type) - with mock.patch(MPATH + 'util.read_dmi_data', return_value='!VMware'): + with mock.patch(MPATH + 'dmi.read_dmi_data', return_value='!VMware'): with mock.patch(MPATH + 'transport_vmware_guestinfo') as m_guestd: with mock.patch(MPATH + 'transport_iso9660') as m_iso9660: m_iso9660.return_value = NOT_FOUND @@ -334,7 +334,7 @@ class TestDatasourceOVF(CiTestCase): self.assertEqual('ovf', ds.cloud_name) self.assertEqual('ovf', ds.platform_type) - with mock.patch(MPATH + 'util.read_dmi_data', return_value='VMWare'): + with mock.patch(MPATH + 'dmi.read_dmi_data', return_value='VMWare'): with mock.patch(MPATH + 'transport_vmware_guestinfo') as m_guestd: with mock.patch(MPATH + 'transport_iso9660') as m_iso9660: m_iso9660.return_value = NOT_FOUND diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 9d82bda9..32f3274a 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -87,7 +87,7 @@ class TestOnScaleway(CiTestCase): @mock.patch('cloudinit.util.get_cmdline') @mock.patch('os.path.exists') - @mock.patch('cloudinit.util.read_dmi_data') + @mock.patch('cloudinit.dmi.read_dmi_data') def test_not_on_scaleway(self, m_read_dmi_data, m_file_exists, m_get_cmdline): self.install_mocks( @@ -105,7 +105,7 @@ class TestOnScaleway(CiTestCase): @mock.patch('cloudinit.util.get_cmdline') @mock.patch('os.path.exists') - @mock.patch('cloudinit.util.read_dmi_data') + @mock.patch('cloudinit.dmi.read_dmi_data') def test_on_scaleway_dmi(self, m_read_dmi_data, m_file_exists, m_get_cmdline): """ @@ -121,7 +121,7 @@ class TestOnScaleway(CiTestCase): @mock.patch('cloudinit.util.get_cmdline') @mock.patch('os.path.exists') - @mock.patch('cloudinit.util.read_dmi_data') + @mock.patch('cloudinit.dmi.read_dmi_data') def test_on_scaleway_var_run_scaleway(self, m_read_dmi_data, m_file_exists, m_get_cmdline): """ @@ -136,7 +136,7 @@ class TestOnScaleway(CiTestCase): @mock.patch('cloudinit.util.get_cmdline') @mock.patch('os.path.exists') - @mock.patch('cloudinit.util.read_dmi_data') + @mock.patch('cloudinit.dmi.read_dmi_data') def test_on_scaleway_cmdline(self, m_read_dmi_data, m_file_exists, m_get_cmdline): """ diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index cca53123..857629f1 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -492,129 +492,6 @@ class TestIsX86(helpers.CiTestCase): self.assertTrue(util.is_x86()) -class TestReadDMIData(helpers.FilesystemMockingTestCase): - - def setUp(self): - super(TestReadDMIData, self).setUp() - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - p = mock.patch("cloudinit.util.is_container", return_value=False) - self.addCleanup(p.stop) - self._m_is_container = p.start() - - def _create_sysfs_parent_directory(self): - util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) - - def _create_sysfs_file(self, key, content): - """Mocks the sys path found on Linux systems.""" - self._create_sysfs_parent_directory() - dmi_key = "/sys/class/dmi/id/{0}".format(key) - util.write_file(dmi_key, content) - - def _configure_dmidecode_return(self, key, content, error=None): - """ - In order to test a missing sys path and call outs to dmidecode, this - function fakes the results of dmidecode to test the results. - """ - def _dmidecode_subp(cmd): - if cmd[-1] != key: - raise subp.ProcessExecutionError() - return (content, error) - - self.patched_funcs.enter_context( - mock.patch("cloudinit.subp.which", side_effect=lambda _: True)) - self.patched_funcs.enter_context( - mock.patch("cloudinit.subp.subp", side_effect=_dmidecode_subp)) - - def patch_mapping(self, new_mapping): - self.patched_funcs.enter_context( - mock.patch('cloudinit.util.DMIDECODE_TO_DMI_SYS_MAPPING', - new_mapping)) - - def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): - self.patch_mapping({'mapped-key': 'mapped-value'}) - expected_dmi_value = 'sys-used-correctly' - self._create_sysfs_file('mapped-value', expected_dmi_value) - self._configure_dmidecode_return('mapped-key', 'wrong-wrong-wrong') - self.assertEqual(expected_dmi_value, util.read_dmi_data('mapped-key')) - - def test_dmidecode_used_if_no_sysfs_file_on_disk(self): - self.patch_mapping({}) - self._create_sysfs_parent_directory() - expected_dmi_value = 'dmidecode-used' - self._configure_dmidecode_return('use-dmidecode', expected_dmi_value) - with mock.patch("cloudinit.util.os.uname") as m_uname: - m_uname.return_value = ('x-sysname', 'x-nodename', - 'x-release', 'x-version', 'x86_64') - self.assertEqual(expected_dmi_value, - util.read_dmi_data('use-dmidecode')) - - def test_dmidecode_not_used_on_arm(self): - self.patch_mapping({}) - print("current =%s", subp) - self._create_sysfs_parent_directory() - dmi_val = 'from-dmidecode' - dmi_name = 'use-dmidecode' - self._configure_dmidecode_return(dmi_name, dmi_val) - print("now =%s", subp) - - expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val} - found = {} - # we do not run the 'dmi-decode' binary on some arches - # verify that anything requested that is not in the sysfs dir - # will return None on those arches. - with mock.patch("cloudinit.util.os.uname") as m_uname: - for arch in expected: - m_uname.return_value = ('x-sysname', 'x-nodename', - 'x-release', 'x-version', arch) - print("now2 =%s", subp) - found[arch] = util.read_dmi_data(dmi_name) - self.assertEqual(expected, found) - - def test_none_returned_if_neither_source_has_data(self): - self.patch_mapping({}) - self._configure_dmidecode_return('key', 'value') - self.assertIsNone(util.read_dmi_data('expect-fail')) - - def test_none_returned_if_dmidecode_not_in_path(self): - self.patched_funcs.enter_context( - mock.patch.object(subp, 'which', lambda _: False)) - self.patch_mapping({}) - self.assertIsNone(util.read_dmi_data('expect-fail')) - - def test_dots_returned_instead_of_foxfox(self): - # uninitialized dmi values show as \xff, return those as . - my_len = 32 - dmi_value = b'\xff' * my_len + b'\n' - expected = "" - dmi_key = 'system-product-name' - sysfs_key = 'product_name' - self._create_sysfs_file(sysfs_key, dmi_value) - self.assertEqual(expected, util.read_dmi_data(dmi_key)) - - def test_container_returns_none(self): - """In a container read_dmi_data should always return None.""" - - # first verify we get the value if not in container - self._m_is_container.return_value = False - key, val = ("system-product-name", "my_product") - self._create_sysfs_file('product_name', val) - self.assertEqual(val, util.read_dmi_data(key)) - - # then verify in container returns None - self._m_is_container.return_value = True - self.assertIsNone(util.read_dmi_data(key)) - - def test_container_returns_none_on_unknown(self): - """In a container even bogus keys return None.""" - self._m_is_container.return_value = True - self._create_sysfs_file('product_name', "should-be-ignored") - self.assertIsNone(util.read_dmi_data("bogus")) - self.assertIsNone(util.read_dmi_data("system-product-name")) - - class TestGetConfigLogfiles(helpers.CiTestCase): def test_empty_cfg_returns_empty_list(self): -- cgit v1.2.3 From 2c4500394b2e3b3074468fffc4d91b404304bc39 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Mon, 2 Nov 2020 22:13:02 +0100 Subject: remove unnecessary reboot from gpart resize (#646) a reboot after `gpart resize` hasn't been necessary since ca FreeBSD 9.2 Co-authored-by: Rick Harding --- cloudinit/config/cc_growpart.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 237c3d02..621df0ac 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -196,10 +196,6 @@ class ResizeGpart(object): util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) from e - # Since growing the FS requires a reboot, make sure we reboot - # first when this module has finished. - open('/var/run/reboot-required', 'a').close() - return (before, get_size(partdev)) -- cgit v1.2.3 From 7978feb3af5846a1a2ac489cde83febe975b046d Mon Sep 17 00:00:00 2001 From: WebSpider Date: Tue, 3 Nov 2020 15:47:03 +0100 Subject: Fix not sourcing default 50-cloud-init ENI file on Debian (#598) * Include both Ubuntu-style cfg file, and Debian-style directory in ENI * Add WebSpider as contributor --- cloudinit/sources/helpers/vmware/imc/config_nic.py | 1 + tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 3745a262..9cd2c0c0 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -275,6 +275,7 @@ class NicConfigurator(object): "# DO NOT EDIT THIS FILE BY HAND --" " AUTOMATICALLY GENERATED BY cloud-init", "source /etc/network/interfaces.d/*.cfg", + "source-directory /etc/network/interfaces.d", ] util.write_file(interfaceFile, content='\n'.join(lines)) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 2d81b700..fc05f982 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -26,3 +26,4 @@ sshedi TheRealFalcon tomponline tsanghan +WebSpider -- cgit v1.2.3 From 0184e53648ca62072c32de3f0de3cc7c91c27d99 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 3 Nov 2020 10:36:48 -0500 Subject: Revert ".travis.yml: use a known-working version of lxd (#643)" (#650) This reverts commit f72d0cb7a03cc311700ea8795edf2454172c7414. A new release of lxd has been released to the stable channel, which includes a fix for the issue we were previously working around. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4342aacb..87841424 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: # pylxd talks only to the lxd from snap - sudo apt remove --purge lxd lxd-client - sudo rm -Rf /var/lib/lxd - - sudo snap install --channel=4.6/stable lxd + - sudo snap install lxd - sudo lxd init --auto - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles # Move any cached lxd images into lxd's image dir -- cgit v1.2.3 From 8dfd8801ab1329efe066876c037f71a73dcf3de1 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Nov 2020 12:12:42 -0500 Subject: Make some language improvements in growpart documentation (#649) * Fix awkward English in sentence * Add the missing word "the" * Fix misspelling * Add @jsf9k as a contributor Co-authored-by: Rick Harding --- cloudinit/config/cc_growpart.py | 11 ++++++----- tools/.github-cla-signers | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 621df0ac..9f338ad1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -16,12 +16,13 @@ This is useful for cloud instances with a larger amount of disk space available than the pristine image uses, as it allows the instance to automatically make use of the extra space. -The devices run growpart on are specified as a list under the ``devices`` key. -Each entry in the devices list can be either the path to the device's -mountpoint in the filesystem or a path to the block device in ``/dev``. +The devices on which to run growpart are specified as a list under the +``devices`` key. Each entry in the devices list can be either the path to the +device's mountpoint in the filesystem or a path to the block device in +``/dev``. The utility to use for resizing can be selected using the ``mode`` config key. -If ``mode`` key is set to ``auto``, then any available utility (either +If the ``mode`` key is set to ``auto``, then any available utility (either ``growpart`` or BSD ``gpart``) will be used. If neither utility is available, no error will be raised. If ``mode`` is set to ``growpart``, then the ``growpart`` utility will be used. If this utility is not available on the @@ -34,7 +35,7 @@ where one tool is able to function and the other is not. The default configuration for both should work for most cloud instances. To explicitly prevent ``cloud-initramfs-tools`` from running ``growroot``, the file ``/etc/growroot-disabled`` can be created. By default, both ``growroot`` and -``cc_growpart`` will check for the existance of this file and will not run if +``cc_growpart`` will check for the existence of this file and will not run if it is present. However, this file can be ignored for ``cc_growpart`` by setting ``ignore_growroot_disabled`` to ``true``. For more information on ``cloud-initramfs-tools`` see: https://launchpad.net/cloud-initramfs-tools diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index fc05f982..2090dc2a 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -12,6 +12,7 @@ emmanuelthome izzyleung johnsonshi jqueuniet +jsf9k landon912 lucasmoura manuelisimo -- cgit v1.2.3 From d1c01c1df94e07afd3843e31b1e1ce670cf07387 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 4 Nov 2020 08:31:10 -0500 Subject: PULL_REQUEST_TEMPLATE.md: expand commit message section (#642) Make it a little easier for people to understand what we're looking for in a commit message. --- .github/PULL_REQUEST_TEMPLATE.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4314b9da..2b59d10a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,18 @@ ## Proposed Commit Message +> summary: no more than 70 characters +> +> A description of what the change being made is and why it is being +> made, if the summary line is insufficient. The blank line above is +> required. This should be wrapped at 72 characters, but otherwise has +> no particular length requirements. +> +> If you need to write multiple paragraphs, feel free. +> +> LP: #NNNNNNN (replace with the appropriate bug reference or remove +> this line entirely if there is no associated bug) + ## Additional Context -- cgit v1.2.3 From cf6c36a18de233190996f7c2dde1e04908bad50a Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Wed, 4 Nov 2020 15:49:34 +0100 Subject: split read_fs_info into linux & freebsd parts (#625) FreeBSD doesn't have blkid, so we want to use geom to list devices and their fstypes and labels. This PR also adds `jail` to the list of is_container() And we now also properly cache geom and blkid output! A test is added to verify the new behaviour by correctly identifying NoCloud on FreeBSD. Co-authored-by: Scott Moser --- tests/unittests/test_ds_identify.py | 40 ++++++++++++++ tools/ds-identify | 106 +++++++++++++++++++++++++++++++++--- 2 files changed, 137 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 9314b244..5f8a4a29 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -20,6 +20,8 @@ UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " UNAME_PPC64EL = ("Linux diamond 4.4.0-83-generic #106-Ubuntu SMP " "Mon Jun 26 17:53:54 UTC 2017 " "ppc64le ppc64le ppc64le GNU/Linux") +UNAME_FREEBSD = ("FreeBSD fbsd12-1 12.1-RELEASE-p10 " + "FreeBSD 12.1-RELEASE-p10 GENERIC amd64") BLKID_EFI_ROOT = """ DEVNAME=/dev/sda1 @@ -80,6 +82,7 @@ MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} MOCK_VIRT_IS_VM_OTHER = {'name': 'detect_virt', 'RET': 'vm-other', 'ret': 0} MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} +MOCK_UNAME_IS_FREEBSD = {'name': 'uname', 'out': UNAME_FREEBSD, 'ret': 0} shell_true = 0 shell_false = 1 @@ -257,6 +260,10 @@ class TestDsIdentify(DsIdentifyBase): """EC2: bobrightbox.com in product_serial is not brightbox'""" self._test_ds_not_found('Ec2-brightbox-negative') + def test_freebsd_nocloud(self): + """NoCloud identified on FreeBSD via label by geom.""" + self._test_ds_found('NoCloud-fbsd') + def test_gce_by_product_name(self): """GCE identifies itself with product_name.""" self._test_ds_found('GCE') @@ -725,6 +732,26 @@ def blkid_out(disks=None): return '\n'.join(lines) +def geom_out(disks=None): + """Convert a list of disk dictionaries into geom content. + + geom called with -a (provider) and -s (script-friendly), will produce the + following output: + + gpt/gptboot0 N/A vtbd1p1 + gpt/swap0 N/A vtbd1p2 + iso9660/cidata N/A vtbd2 + """ + if disks is None: + disks = [] + lines = [] + for disk in disks: + lines.append("%s/%s N/A %s" % ( + disk["TYPE"], disk["LABEL"], disk["DEVNAME"])) + lines.append("") + return '\n'.join(lines) + + def _print_run_output(rc, out, err, cfg, files): """A helper to print return of TestDsIdentify. @@ -807,6 +834,19 @@ VALID_CFG = { 'dev/vdb': 'pretend iso content for cidata\n', } }, + 'NoCloud-fbsd': { + 'ds': 'NoCloud', + 'mocks': [ + MOCK_VIRT_IS_KVM, + MOCK_UNAME_IS_FREEBSD, + {'name': 'geom', 'ret': 0, + 'out': geom_out( + [{'DEVNAME': 'vtbd', 'TYPE': 'iso9660', 'LABEL': 'cidata'}])}, + ], + 'files': { + '/dev/vtdb': 'pretend iso content for cidata\n', + } + }, 'NoCloudUpper': { 'ds': 'NoCloud', 'mocks': [ diff --git a/tools/ds-identify b/tools/ds-identify index 4e5700fc..ec9e775e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -92,7 +92,8 @@ _DI_LOGGED="" # set DI_MAIN='noop' in environment to source this file with no main called. DI_MAIN=${DI_MAIN:-main} -DI_BLKID_OUTPUT="" +DI_BLKID_EXPORT_OUT="" +DI_GEOM_LABEL_STATUS_OUT="" DI_DEFAULT_POLICY="search,found=all,maybe=all,notfound=${DI_DISABLED}" DI_DEFAULT_POLICY_NO_DMI="search,found=all,maybe=all,notfound=${DI_ENABLED}" DI_DMI_CHASSIS_ASSET_TAG="" @@ -231,8 +232,19 @@ ensure_sane_path() { done } -read_fs_info() { - cached "${DI_BLKID_OUTPUT}" && return 0 +blkid_export() { + # call 'blkid -c /dev/null export', set DI_BLKID_EXPORT_OUT + cached "$DI_BLKID_EXPORT_OUT" && return 0 + local out="" ret=0 + out=$(blkid -c /dev/null -o export) && DI_BLKID_EXPORT_OUT="$out" || { + ret=$? + error "failed running [$ret]: blkid -c /dev/null -o export" + DI_BLKID_EXPORT_OUT="$UNAVAILABLE" + } + return $ret +} + +read_fs_info_linux() { # do not rely on links in /dev/disk which might not be present yet. # Note that blkid < 2.22 (centos6, trusty) do not output DEVNAME. # that means that DI_ISO9660_DEVS will not be set. @@ -244,20 +256,23 @@ read_fs_info() { return fi local oifs="$IFS" line="" delim="," - local ret=0 out="" labels="" dev="" label="" ftype="" isodevs="" uuids="" - out=$(blkid -c /dev/null -o export) || { - ret=$? - error "failed running [$ret]: blkid -c /dev/null -o export" + local ret=0 labels="" dev="" label="" ftype="" isodevs="" uuids="" + + blkid_export + ret=$? + [ "$DI_BLKID_EXPORT_OUT" = "$UNAVAILABLE" ] && { DI_FS_LABELS="$UNAVAILABLE:error" DI_ISO9660_DEVS="$UNAVAILABLE:error" + DI_FS_UUIDS="$UNAVAILABLE:error" return $ret } + # 'set --' will collapse multiple consecutive entries in IFS for # whitespace characters (\n, tab, " ") so we cannot rely on getting # empty lines in "$@" below. # shellcheck disable=2086 - { IFS="$CR"; set -- $out; IFS="$oifs"; } + { IFS="$CR"; set -- $DI_BLKID_EXPORT_OUT; IFS="$oifs"; } for line in "$@"; do case "${line}" in @@ -281,6 +296,74 @@ read_fs_info() { DI_ISO9660_DEVS="${isodevs#,}" } +geom_label_status_as() { + # call 'geom label status -as', set DI_GEOM_LABEL_STATUS_OUT + cached "$DI_GEOM_LABEL_STATUS_OUT" && return 0 + local out="" ret=0 + out=$(geom label status -as) && DI_GEOM_LABEL_STATUS_OUT="$out" || { + ret=$? + error "failed running [$ret]: geom label status -as" + DI_GEOM_LABEL_STATUS_OUT="$UNAVAILABLE" + } + return $ret +} + + +read_fs_info_freebsd() { + local oifs="$IFS" line="" delim="," + local ret=0 labels="" dev="" label="" ftype="" isodevs="" + + geom_label_status_as + ret=$? + [ "$DI_GEOM_LABEL_STATUS_OUT" = "$UNAVAILABLE" ] && { + DI_FS_LABELS="$UNAVAILABLE:error" + DI_ISO9660_DEVS="$UNAVAILABLE:error" + return $ret + } + + # The expected output looks like this: + # gpt/gptboot0 N/A vtbd1p1 + # gpt/swap0 N/A vtbd1p2 + # iso9660/cidata N/A vtbd2 + + # shellcheck disable=2086 + { IFS="$CR"; set -- $DI_GEOM_LABEL_STATUS_OUT; IFS="$oifs"; } + + for line in "$@"; do + # shellcheck disable=2086 + set -- $line + provider=$1 + ftype="${provider%/*}" + label="${provider#*/}" + dev=$3 + + [ -n "$dev" -a "$ftype" = "iso9660" ] && + isodevs="${isodevs},${dev}=$label" + + labels="${labels}${label}${delim}" + done + + DI_FS_LABELS="${labels%${delim}}" + DI_ISO9660_DEVS="${isodevs#,}" +} + +read_fs_info() { + # After calling its subfunctions, read_fs_info() will set the following + # variables: + # + # - DI_FS_LABELS + # - DI_ISO9660_DEVS + # - DI_FS_UUIDS + + if [ "$DI_UNAME_KERNEL_NAME" = "FreeBSD" ]; then + read_fs_info_freebsd + return $? + else + read_fs_info_linux + return $? + fi +} + cached() { [ -n "$1" ] && _RET="$1" && return || return 1 } @@ -319,6 +402,11 @@ detect_virt() { *) virt="$out" esac } + out=$(sysctl -qn security.jail.jailed 2>/dev/null) && { + if [ "$out" = "1" ]; then + virt="jail" + fi + } fi _RET="$virt" } @@ -331,7 +419,7 @@ read_virt() { is_container() { case "${DI_VIRT}" in - container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;; + container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt|jail) return 0;; *) return 1;; esac } -- cgit v1.2.3 From f42359741a6446d3c4ef25266ec290ed6f925b3a Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 4 Nov 2020 11:54:20 -0600 Subject: Fix launch_kwargs bug in integration tests (#654) Integration test instance launch would previously fail if provided launch_kwargs is None --- tests/integration_tests/clouds.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 08c86198..71d4e85d 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -59,7 +59,8 @@ class IntegrationCloud(ABC): 'user_data': user_data, 'wait': False, } - kwargs.update(launch_kwargs) + if launch_kwargs: + kwargs.update(launch_kwargs) log.info( "Launching instance with launch_kwargs:\n{}".format( "\n".join("{}={}".format(*item) for item in kwargs.items()) -- cgit v1.2.3 From c86283f0d9fe8a2634dc3c47727e6218fdaf25e2 Mon Sep 17 00:00:00 2001 From: Moustafa Moustafa Date: Wed, 4 Nov 2020 11:51:16 -0800 Subject: azure: enable pushing the log to KVP from the last pushed byte (#614) This allows the cloud-init log to be pushed multiple times during boot, with the latest lines being pushed each time. --- cloudinit/sources/helpers/azure.py | 44 +++++++++++++++++++++++--------- tests/unittests/test_reporting_hyperv.py | 7 ++--- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 560cadba..4071a50e 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -9,6 +9,7 @@ import struct import time import textwrap import zlib +from errno import ENOENT from cloudinit.settings import CFG_BUILTIN from cloudinit.net import dhcp @@ -41,8 +42,9 @@ COMPRESSED_EVENT_TYPE = 'compressed' # cloud-init.log files where the P95 of the file sizes was 537KB and the time # consumed to dump 500KB file was (P95:76, P99:233, P99.9:1170) in ms MAX_LOG_TO_KVP_LENGTH = 512000 -# Marker file to indicate whether cloud-init.log is pushed to KVP -LOG_PUSHED_TO_KVP_MARKER_FILE = '/var/lib/cloud/data/log_pushed_to_kvp' +# File to store the last byte of cloud-init.log that was pushed to KVP. This +# file will be deleted with every VM reboot. +LOG_PUSHED_TO_KVP_INDEX_FILE = '/run/cloud-init/log_pushed_to_kvp_index' azure_ds_reporter = events.ReportEventStack( name="azure-ds", description="initialize reporter for azure ds", @@ -214,32 +216,50 @@ def report_compressed_event(event_name, event_content): def push_log_to_kvp(file_name=CFG_BUILTIN['def_log_file']): """Push a portion of cloud-init.log file or the whole file to KVP based on the file size. - If called more than once, it skips pushing the log file to KVP again.""" + The first time this function is called after VM boot, It will push the last + n bytes of the log file such that n < MAX_LOG_TO_KVP_LENGTH + If called again on the same boot, it continues from where it left off.""" - log_pushed_to_kvp = bool(os.path.isfile(LOG_PUSHED_TO_KVP_MARKER_FILE)) - if log_pushed_to_kvp: - report_diagnostic_event( - "cloud-init.log is already pushed to KVP", logger_func=LOG.debug) - return + start_index = get_last_log_byte_pushed_to_kvp_index() LOG.debug("Dumping cloud-init.log file to KVP") try: with open(file_name, "rb") as f: f.seek(0, os.SEEK_END) - seek_index = max(f.tell() - MAX_LOG_TO_KVP_LENGTH, 0) + seek_index = max(f.tell() - MAX_LOG_TO_KVP_LENGTH, start_index) report_diagnostic_event( - "Dumping last {} bytes of cloud-init.log file to KVP".format( - f.tell() - seek_index), + "Dumping last {0} bytes of cloud-init.log file to KVP starting" + " from index: {1}".format(f.tell() - seek_index, seek_index), logger_func=LOG.debug) f.seek(seek_index, os.SEEK_SET) report_compressed_event("cloud-init.log", f.read()) - util.write_file(LOG_PUSHED_TO_KVP_MARKER_FILE, '') + util.write_file(LOG_PUSHED_TO_KVP_INDEX_FILE, str(f.tell())) except Exception as ex: report_diagnostic_event( "Exception when dumping log file: %s" % repr(ex), logger_func=LOG.warning) +@azure_ds_telemetry_reporter +def get_last_log_byte_pushed_to_kvp_index(): + try: + with open(LOG_PUSHED_TO_KVP_INDEX_FILE, "r") as f: + return int(f.read()) + except IOError as e: + if e.errno != ENOENT: + report_diagnostic_event("Reading LOG_PUSHED_TO_KVP_INDEX_FILE" + " failed: %s." % repr(e), + logger_func=LOG.warning) + except ValueError as e: + report_diagnostic_event("Invalid value in LOG_PUSHED_TO_KVP_INDEX_FILE" + ": %s." % repr(e), + logger_func=LOG.warning) + except Exception as e: + report_diagnostic_event("Failed to get the last log byte pushed to KVP" + ": %s." % repr(e), logger_func=LOG.warning) + return 0 + + @contextmanager def cd(newdir): prevdir = os.getcwd() diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index 3f63a60e..8f7b3694 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -237,7 +237,7 @@ class TextKvpReporter(CiTestCase): instantiated_handler_registry.register_item("telemetry", reporter) log_file = self.tmp_path("cloud-init.log") azure.MAX_LOG_TO_KVP_LENGTH = 100 - azure.LOG_PUSHED_TO_KVP_MARKER_FILE = self.tmp_path( + azure.LOG_PUSHED_TO_KVP_INDEX_FILE = self.tmp_path( 'log_pushed_to_kvp') with open(log_file, "w") as f: log_content = "A" * 50 + "B" * 100 @@ -254,8 +254,9 @@ class TextKvpReporter(CiTestCase): self.assertNotEqual( event.event_type, azure.COMPRESSED_EVENT_TYPE) self.validate_compressed_kvps( - reporter, 1, - [log_content[-azure.MAX_LOG_TO_KVP_LENGTH:].encode()]) + reporter, 2, + [log_content[-azure.MAX_LOG_TO_KVP_LENGTH:].encode(), + extra_content.encode()]) finally: instantiated_handler_registry.unregister_item("telemetry", force=False) -- cgit v1.2.3 From b542ce7a7c530fbfa874dd780294628b62fe092a Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 5 Nov 2020 14:24:47 -0600 Subject: Prevent timeout on travis integration tests. (#651) Add a script in travis to output a dot every 10 minute during and integration test run to prevent travis timeout. Co-authored-by: Rick Harding --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 87841424..496c1a81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -199,7 +199,16 @@ matrix: fi # Use sudo to get a new shell where we're in the sbuild group - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' - - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' + - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' & + - | + SECONDS=0 + while [ -e /proc/$! ]; do + if [ "$SECONDS" -gt "570" ]; then + echo -n '.' + SECONDS=0 + fi + sleep 10 + done - python: 3.5 env: TOXENV=xenial -- cgit v1.2.3 From d83c0bb4baca0b57166a74055f410fa4f75a08f5 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Fri, 6 Nov 2020 19:49:05 +0100 Subject: replace usage of dmidecode with kenv on FreeBSD (#621) FreeBSD lets us read out kernel parameters with kenv(1), a user-space utility that's shipped in "base" We can use it in place of dmidecode(8), thus removing the dependency on sysutils/dmidecode, and the restrictions to i386 and x86_64 architectures that this utility imposes on FreeBSD. Co-authored-by: Scott Moser --- cloudinit/dmi.py | 86 +++++++++++++++++++++++++------------ cloudinit/tests/test_dmi.py | 27 +++++++++++- cloudinit/util.py | 55 ++++++++++++++++++------ tests/unittests/test_ds_identify.py | 25 +++++++++-- tools/build-on-freebsd | 1 - tools/ds-identify | 42 ++++++++++++++++-- 6 files changed, 185 insertions(+), 51 deletions(-) diff --git a/cloudinit/dmi.py b/cloudinit/dmi.py index 96e0e423..f0e69a5a 100644 --- a/cloudinit/dmi.py +++ b/cloudinit/dmi.py @@ -1,8 +1,9 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging from cloudinit import subp -from cloudinit.util import is_container +from cloudinit.util import is_container, is_FreeBSD +from collections import namedtuple import os LOG = logging.getLogger(__name__) @@ -10,38 +11,43 @@ LOG = logging.getLogger(__name__) # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" -# dmidecode and /sys/class/dmi/id/* use different names for the same value, -# this allows us to refer to them by one canonical name -DMIDECODE_TO_DMI_SYS_MAPPING = { - 'baseboard-asset-tag': 'board_asset_tag', - 'baseboard-manufacturer': 'board_vendor', - 'baseboard-product-name': 'board_name', - 'baseboard-serial-number': 'board_serial', - 'baseboard-version': 'board_version', - 'bios-release-date': 'bios_date', - 'bios-vendor': 'bios_vendor', - 'bios-version': 'bios_version', - 'chassis-asset-tag': 'chassis_asset_tag', - 'chassis-manufacturer': 'chassis_vendor', - 'chassis-serial-number': 'chassis_serial', - 'chassis-version': 'chassis_version', - 'system-manufacturer': 'sys_vendor', - 'system-product-name': 'product_name', - 'system-serial-number': 'product_serial', - 'system-uuid': 'product_uuid', - 'system-version': 'product_version', +kdmi = namedtuple('KernelNames', ['linux', 'freebsd']) +kdmi.__new__.defaults__ = (None, None) + +# FreeBSD's kenv(1) and Linux /sys/class/dmi/id/* both use different names from +# dmidecode. The values are the same, and ultimately what we're interested in. +# These tools offer a "cheaper" way to access those values over dmidecode. +# This is our canonical translation table. If we add more tools on other +# platforms to find dmidecode's values, their keys need to be put in here. +DMIDECODE_TO_KERNEL = { + 'baseboard-asset-tag': kdmi('board_asset_tag', 'smbios.planar.tag'), + 'baseboard-manufacturer': kdmi('board_vendor', 'smbios.planar.maker'), + 'baseboard-product-name': kdmi('board_name', 'smbios.planar.product'), + 'baseboard-serial-number': kdmi('board_serial', 'smbios.planar.serial'), + 'baseboard-version': kdmi('board_version', 'smbios.planar.version'), + 'bios-release-date': kdmi('bios_date', 'smbios.bios.reldate'), + 'bios-vendor': kdmi('bios_vendor', 'smbios.bios.vendor'), + 'bios-version': kdmi('bios_version', 'smbios.bios.version'), + 'chassis-asset-tag': kdmi('chassis_asset_tag', 'smbios.chassis.tag'), + 'chassis-manufacturer': kdmi('chassis_vendor', 'smbios.chassis.maker'), + 'chassis-serial-number': kdmi('chassis_serial', 'smbios.chassis.serial'), + 'chassis-version': kdmi('chassis_version', 'smbios.chassis.version'), + 'system-manufacturer': kdmi('sys_vendor', 'smbios.system.maker'), + 'system-product-name': kdmi('product_name', 'smbios.system.product'), + 'system-serial-number': kdmi('product_serial', 'smbios.system.serial'), + 'system-uuid': kdmi('product_uuid', 'smbios.system.uuid'), + 'system-version': kdmi('product_version', 'smbios.system.version'), } def _read_dmi_syspath(key): """ - Reads dmi data with from /sys/class/dmi/id + Reads dmi data from /sys/class/dmi/id """ - if key not in DMIDECODE_TO_DMI_SYS_MAPPING: + kmap = DMIDECODE_TO_KERNEL.get(key) + if kmap is None or kmap.linux is None: return None - mapped_key = DMIDECODE_TO_DMI_SYS_MAPPING[key] - dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, mapped_key) - + dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, kmap.linux) LOG.debug("querying dmi data %s", dmi_key_path) if not os.path.exists(dmi_key_path): LOG.debug("did not find %s", dmi_key_path) @@ -68,6 +74,29 @@ def _read_dmi_syspath(key): return None +def _read_kenv(key): + """ + Reads dmi data from FreeBSD's kenv(1) + """ + kmap = DMIDECODE_TO_KERNEL.get(key) + if kmap is None or kmap.freebsd is None: + return None + + LOG.debug("querying dmi data %s", kmap.freebsd) + + try: + cmd = ["kenv", "-q", kmap.freebsd] + (result, _err) = subp.subp(cmd) + result = result.strip() + LOG.debug("kenv returned '%s' for '%s'", result, kmap.freebsd) + return result + except subp.ProcessExecutionError as e: + LOG.debug('failed kenv cmd: %s\n%s', cmd, e) + return None + + return None + + def _call_dmidecode(key, dmidecode_path): """ Calls out to dmidecode to get the data out. This is mostly for supporting @@ -81,7 +110,7 @@ def _call_dmidecode(key, dmidecode_path): if result.replace(".", "") == "": return "" return result - except (IOError, OSError) as e: + except subp.ProcessExecutionError as e: LOG.debug('failed dmidecode cmd: %s\n%s', cmd, e) return None @@ -107,6 +136,9 @@ def read_dmi_data(key): if is_container(): return None + if is_FreeBSD(): + return _read_kenv(key) + syspath_value = _read_dmi_syspath(key) if syspath_value is not None: return syspath_value diff --git a/cloudinit/tests/test_dmi.py b/cloudinit/tests/test_dmi.py index 4a8af257..78a72122 100644 --- a/cloudinit/tests/test_dmi.py +++ b/cloudinit/tests/test_dmi.py @@ -19,6 +19,9 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): p = mock.patch("cloudinit.dmi.is_container", return_value=False) self.addCleanup(p.stop) self._m_is_container = p.start() + p = mock.patch("cloudinit.dmi.is_FreeBSD", return_value=False) + self.addCleanup(p.stop) + self._m_is_FreeBSD = p.start() def _create_sysfs_parent_directory(self): util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) @@ -44,13 +47,26 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self.patched_funcs.enter_context( mock.patch("cloudinit.dmi.subp.subp", side_effect=_dmidecode_subp)) + def _configure_kenv_return(self, key, content, error=None): + """ + In order to test a FreeBSD system call outs to kenv, this + function fakes the results of kenv to test the results. + """ + def _kenv_subp(cmd): + if cmd[-1] != dmi.DMIDECODE_TO_KERNEL[key].freebsd: + raise subp.ProcessExecutionError() + return (content, error) + + self.patched_funcs.enter_context( + mock.patch("cloudinit.dmi.subp.subp", side_effect=_kenv_subp)) + def patch_mapping(self, new_mapping): self.patched_funcs.enter_context( - mock.patch('cloudinit.dmi.DMIDECODE_TO_DMI_SYS_MAPPING', + mock.patch('cloudinit.dmi.DMIDECODE_TO_KERNEL', new_mapping)) def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): - self.patch_mapping({'mapped-key': 'mapped-value'}) + self.patch_mapping({'mapped-key': dmi.kdmi('mapped-value', None)}) expected_dmi_value = 'sys-used-correctly' self._create_sysfs_file('mapped-value', expected_dmi_value) self._configure_dmidecode_return('mapped-key', 'wrong-wrong-wrong') @@ -129,3 +145,10 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self._create_sysfs_file('product_name', "should-be-ignored") self.assertIsNone(dmi.read_dmi_data("bogus")) self.assertIsNone(dmi.read_dmi_data("system-product-name")) + + def test_freebsd_uses_kenv(self): + """On a FreeBSD system, kenv is called.""" + self._m_is_FreeBSD.return_value = True + key, val = ("system-product-name", "my_product") + self._configure_kenv_return(key, val) + self.assertEqual(dmi.read_dmi_data(key), val) diff --git a/cloudinit/util.py b/cloudinit/util.py index bdb3694d..769f3425 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -62,12 +62,6 @@ TRUE_STRINGS = ('true', '1', 'on', 'yes') FALSE_STRINGS = ('off', '0', 'no', 'false') -# Helper utils to see if running in a container -CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], - ['running-in-container'], - ['lxc-is-container']) - - def kernel_version(): return tuple(map(int, os.uname().release.split('.')[:2])) @@ -1928,19 +1922,52 @@ def strip_prefix_suffix(line, prefix=None, suffix=None): return line +def _cmd_exits_zero(cmd): + if subp.which(cmd[0]) is None: + return False + try: + subp.subp(cmd) + except subp.ProcessExecutionError: + return False + return True + + +def _is_container_systemd(): + return _cmd_exits_zero(["systemd-detect-virt", "--quiet", "--container"]) + + +def _is_container_upstart(): + return _cmd_exits_zero(["running-in-container"]) + + +def _is_container_old_lxc(): + return _cmd_exits_zero(["lxc-is-container"]) + + +def _is_container_freebsd(): + if not is_FreeBSD(): + return False + cmd = ["sysctl", "-qn", "security.jail.jailed"] + if subp.which(cmd[0]) is None: + return False + out, _ = subp.subp(cmd) + return out.strip() == "1" + + +@lru_cache() def is_container(): """ Checks to see if this code running in a container of some sort """ - - for helper in CONTAINER_TESTS: - try: - # try to run a helper program. if it returns true/zero - # then we're inside a container. otherwise, no - subp.subp(helper) + checks = ( + _is_container_systemd, + _is_container_freebsd, + _is_container_upstart, + _is_container_old_lxc) + + for helper in checks: + if helper(): return True - except (IOError, OSError): - pass # this code is largely from the logic in # ubuntu's /etc/init/container-detect.conf diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 5f8a4a29..1d8aaf18 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -146,6 +146,8 @@ class DsIdentifyBase(CiTestCase): 'out': 'No value found', 'ret': 1}, {'name': 'dmi_decode', 'ret': 1, 'err': 'No dmidecode program. ERROR.'}, + {'name': 'get_kenv_field', 'ret': 1, + 'err': 'No kenv program. ERROR.'}, ] written = [d['name'] for d in mocks] @@ -651,14 +653,22 @@ class TestDsIdentify(DsIdentifyBase): class TestBSDNoSys(DsIdentifyBase): """Test *BSD code paths - FreeBSD doesn't have /sys so we use dmidecode(8) here - It also doesn't have systemd-detect-virt(8), so we use sysctl(8) to query + FreeBSD doesn't have /sys so we use kenv(1) here. + Other BSD systems fallback to dmidecode(8). + BSDs also doesn't have systemd-detect-virt(8), so we use sysctl(8) to query kern.vm_guest, and optionally map it""" - def test_dmi_decode(self): + def test_dmi_kenv(self): + """Test that kenv(1) works on systems which don't have /sys + + This will be used on FreeBSD systems. + """ + self._test_ds_found('Hetzner-kenv') + + def test_dmi_dmidecode(self): """Test that dmidecode(8) works on systems which don't have /sys - This will be used on *BSD systems. + This will be used on all other BSD systems. """ self._test_ds_found('Hetzner-dmidecode') @@ -1026,6 +1036,13 @@ VALID_CFG = { 'ds': 'Hetzner', 'files': {P_SYS_VENDOR: 'Hetzner\n'}, }, + 'Hetzner-kenv': { + 'ds': 'Hetzner', + 'mocks': [ + MOCK_UNAME_IS_FREEBSD, + {'name': 'get_kenv_field', 'ret': 0, 'RET': 'Hetzner'} + ], + }, 'Hetzner-dmidecode': { 'ds': 'Hetzner', 'mocks': [ diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 94b03433..1e876905 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -21,7 +21,6 @@ py_prefix=$(${PYTHON} -c 'import sys; print("py%d%d" % (sys.version_info.major, depschecked=/tmp/c-i.dependencieschecked pkgs=" bash - dmidecode e2fsprogs $py_prefix-Jinja2 $py_prefix-boto diff --git a/tools/ds-identify b/tools/ds-identify index ec9e775e..496dbb8a 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -180,13 +180,43 @@ debug() { echo "$@" 1>&3 } +get_kenv_field() { + local sys_field="$1" kenv_field="" val="" + command -v kenv >/dev/null 2>&1 || { + warn "No kenv program. Cannot read $sys_field." + return 1 + } + case "$sys_field" in + board_asset_tag) kenv_field="smbios.planar.tag";; + board_vendor) kenv_field='smbios.planar.maker';; + board_name) kenv_field='smbios.planar.product';; + board_serial) kenv_field='smbios.planar.serial';; + board_version) kenv_field='smbios.planar.version';; + bios_date) kenv_field='smbios.bios.reldate';; + bios_vendor) kenv_field='smbios.bios.vendor';; + bios_version) kenv_field='smbios.bios.version';; + chassis_asset_tag) kenv_field='smbios.chassis.tag';; + chassis_vendor) kenv_field='smbios.chassis.maker';; + chassis_serial) kenv_field='smbios.chassis.serial';; + chassis_version) kenv_field='smbios.chassis.version';; + sys_vendor) kenv_field='smbios.system.maker';; + product_name) kenv_field='smbios.system.product';; + product_serial) kenv_field='smbios.system.serial';; + product_uuid) kenv_field='smbios.system.uuid';; + *) error "Unknown field $sys_field. Cannot call kenv." + return 1;; + esac + val=$(kenv -q "$kenv_field" 2>/dev/null) || return 1 + _RET="$val" +} + dmi_decode() { local sys_field="$1" dmi_field="" val="" command -v dmidecode >/dev/null 2>&1 || { warn "No dmidecode program. Cannot read $sys_field." return 1 } - case "$1" in + case "$sys_field" in sys_vendor) dmi_field="system-manufacturer";; product_name) dmi_field="system-product-name";; product_uuid) dmi_field="system-uuid";; @@ -200,8 +230,14 @@ dmi_decode() { } get_dmi_field() { - local path="${PATH_SYS_CLASS_DMI_ID}/$1" _RET="$UNAVAILABLE" + + if [ "$DI_UNAME_KERNEL_NAME" = "FreeBSD" ]; then + get_kenv_field "$1" || _RET="$ERROR" + return $? + fi + + local path="${PATH_SYS_CLASS_DMI_ID}/$1" if [ -d "${PATH_SYS_CLASS_DMI_ID}" ]; then if [ -f "$path" ] && [ -r "$path" ]; then read _RET < "${path}" || _RET="$ERROR" @@ -1310,10 +1346,10 @@ dscheck_IBMCloud() { } collect_info() { + read_uname_info read_virt read_pid1_product_name read_kernel_cmdline - read_uname_info read_config read_datasource_list read_dmi_sys_vendor -- cgit v1.2.3 From 0dc2d0304a7fa33af18475b99e808fbcdcad44f2 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Fri, 6 Nov 2020 21:39:13 +0100 Subject: HACKING.md: unify network-refactoring namespace (#658) We had some inconsistencies between `distro.net.` and `distro.networking.`. This commit unifies all such instances to `distro.networking.`. --- HACKING.rst | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index 4ae7f7b4..8a12e3e3 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -391,9 +391,9 @@ will reference. These will capture the differences between networking on our various distros, while still allowing easy reuse of code between distros that share functionality (e.g. most of the Linux networking behaviour). ``Distro`` objects will instantiate the networking classes -at ``self.net``, so callers will call ``distro.net.`` instead of -``cloudinit.net.``; this will necessitate access to an -instantiated ``Distro`` object. +at ``self.networking``, so callers will call +``distro.networking.`` instead of ``cloudinit.net.``; this +will necessitate access to an instantiated ``Distro`` object. An implementation note: there may be external consumers of the ``cloudinit.net`` module. We don't consider this a public API, so we @@ -426,17 +426,18 @@ In more detail: networking class (initially this will be either ``LinuxNetworking`` or ``BSDNetworking``). * When ``Distro`` classes are instantiated, they will instantiate - ``cls.networking_cls`` and store the instance at ``self.net``. (This - will be implemented in ``cloudinit.distros.Distro.__init__``.) + ``cls.networking_cls`` and store the instance at ``self.networking``. + (This will be implemented in ``cloudinit.distros.Distro.__init__``.) * A helper function will be added which will determine the appropriate ``Distro`` subclass for the current system, instantiate it and return - its ``net`` attribute. (This is the entry point for existing + its ``networking`` attribute. (This is the entry point for existing consumers to migrate to.) * Callers of refactored functions will change from calling - ``cloudinit.net.`` to ``distro.net.``, where ``distro`` - is an instance of the appropriate ``Distro`` class for this system. - (This will require making such an instance available to callers, - which will constitute a large part of the work in this project.) + ``cloudinit.net.`` to ``distro.networking.``, where + ``distro`` is an instance of the appropriate ``Distro`` class for + this system. (This will require making such an instance available to + callers, which will constitute a large part of the work in this + project.) After the initial structure is in place, the work in this refactor will consist of replacing the ``cloudinit.net.some_func`` call in each @@ -448,11 +449,11 @@ time: * find it in the `the list of bugs tagged net-refactor`_ and assign yourself to it (see :ref:`Managing Work/Tracking Progress` below for more details) -* refactor all of its callers to call the ``distro.net.`` method - on ``Distro`` instead of the ``cloudinit.net.`` function. (This - is likely to be the most time-consuming step, as it may require - plumbing ``Distro`` objects through to places that previously have - not consumed them.) +* refactor all of its callers to call the ``distro.networking.`` + method on ``Distro`` instead of the ``cloudinit.net.`` + function. (This is likely to be the most time-consuming step, as it + may require plumbing ``Distro`` objects through to places that + previously have not consumed them.) * refactor its implementation from ``cloudinit.net`` into the ``Networking`` hierarchy (e.g. if it has an if/else on BSD, this is the time to put the implementations in their respective subclasses) -- cgit v1.2.3 From 57349eb7df1c422d9e9558e54b201c85778997ae Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Mon, 9 Nov 2020 17:24:55 +0000 Subject: Make wakeonlan Network Config v2 setting actually work (#626) Add code so that specifying "wakeonlan: true" actually results in relevant configuration entry appearing in /etc/network/interfaces, Netplan, and sysconfig for RHEL and OpenSuse. Add testcases for the above. --- cloudinit/net/eni.py | 4 ++ cloudinit/net/network_state.py | 6 +- cloudinit/net/sysconfig.py | 5 ++ tests/unittests/test_net.py | 143 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 13c041f3..0074691b 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -401,6 +401,10 @@ class Renderer(renderer.Renderer): sections = [] subnets = iface.get('subnets', {}) accept_ra = iface.pop('accept-ra', None) + ethernet_wol = iface.pop('wakeonlan', None) + if ethernet_wol: + # Specify WOL setting 'g' for using "Magic Packet" + iface['ethernet-wol'] = 'g' if subnets: for index, subnet in enumerate(subnets): ipv4_subnet_mtu = None diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index d9e7fd58..e8bf9e39 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -369,6 +369,9 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): accept_ra = command.get('accept-ra', None) if accept_ra is not None: accept_ra = util.is_true(accept_ra) + wakeonlan = command.get('wakeonlan', None) + if wakeonlan is not None: + wakeonlan = util.is_true(wakeonlan) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -379,7 +382,8 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): 'address': None, 'gateway': None, 'subnets': subnets, - 'accept-ra': accept_ra + 'accept-ra': accept_ra, + 'wakeonlan': wakeonlan, }) self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index b0eecc44..a930e612 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -367,6 +367,11 @@ class Renderer(renderer.Renderer): if new_key: iface_cfg[new_key] = old_value + # only set WakeOnLan for physical interfaces + if ('wakeonlan' in iface and iface['wakeonlan'] and + iface['type'] == 'physical'): + iface_cfg['ETHTOOL_OPTS'] = 'wol g' + @classmethod def _render_subnets(cls, iface_cfg, subnets, has_default_route, flavor): # setting base values diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5af82e20..70453683 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1378,6 +1378,89 @@ NETWORK_CONFIGS = { """), }, }, + 'wakeonlan_disabled': { + 'expected_eni': textwrap.dedent("""\ + auto lo + iface lo inet loopback + + auto iface0 + iface iface0 inet dhcp + """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + ethernets: + iface0: + dhcp4: true + wakeonlan: false + version: 2 + """), + 'expected_sysconfig_opensuse': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=dhcp4 + STARTMODE=auto + """), + }, + 'expected_sysconfig_rhel': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=dhcp + DEVICE=iface0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + }, + 'yaml_v2': textwrap.dedent("""\ + version: 2 + ethernets: + iface0: + dhcp4: true + wakeonlan: false + """).rstrip(' '), + }, + 'wakeonlan_enabled': { + 'expected_eni': textwrap.dedent("""\ + auto lo + iface lo inet loopback + + auto iface0 + iface iface0 inet dhcp + ethernet-wol g + """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + ethernets: + iface0: + dhcp4: true + wakeonlan: true + version: 2 + """), + 'expected_sysconfig_opensuse': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=dhcp4 + ETHTOOL_OPTS="wol g" + STARTMODE=auto + """), + }, + 'expected_sysconfig_rhel': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=dhcp + DEVICE=iface0 + ETHTOOL_OPTS="wol g" + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + }, + 'yaml_v2': textwrap.dedent("""\ + version: 2 + ethernets: + iface0: + dhcp4: true + wakeonlan: true + """).rstrip(' '), + }, 'all': { 'expected_eni': ("""\ auto lo @@ -3293,6 +3376,20 @@ USERCTL=no self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) + def test_wakeonlan_disabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_disabled'] + found = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_wakeonlan_enabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_enabled'] + found = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + def test_check_ifcfg_rh(self): """ifcfg-rh plugin is added NetworkManager.conf if conf present.""" render_dir = self.tmp_dir() @@ -3829,6 +3926,20 @@ STARTMODE=auto self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) + def test_wakeonlan_disabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_disabled'] + found = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_wakeonlan_enabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_enabled'] + found = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + def test_render_v4_and_v6(self): entry = NETWORK_CONFIGS['v4_and_v6'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) @@ -4478,6 +4589,22 @@ class TestNetplanRoundTrip(CiTestCase): entry['expected_netplan'].splitlines(), files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_wakeonlan_disabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_disabled'] + files = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_wakeonlan_enabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_enabled'] + files = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_all(self): entry = NETWORK_CONFIGS['all'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) @@ -4645,6 +4772,22 @@ class TestEniRoundTrip(CiTestCase): entry['expected_eni'].splitlines(), files['/etc/network/interfaces'].splitlines()) + def testsimple_wakeonlan_disabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_disabled'] + files = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self.assertEqual( + entry['expected_eni'].splitlines(), + files['/etc/network/interfaces'].splitlines()) + + def testsimple_wakeonlan_enabled_config_v2(self): + entry = NETWORK_CONFIGS['wakeonlan_enabled'] + files = self._render_and_read(network_config=yaml.load( + entry['yaml_v2'])) + self.assertEqual( + entry['expected_eni'].splitlines(), + files['/etc/network/interfaces'].splitlines()) + def testsimple_render_manual(self): """Test rendering of 'manual' for 'type' and 'control'. -- cgit v1.2.3 From 2730521fd566f855863c5ed049a1df26abcd0770 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Nov 2020 09:51:16 -0500 Subject: Fix stacktrace in DataSourceRbxCloud if no metadata disk is found (#632) Largely speaking, ds-identify protects from this scenario being hit, but if DataSourceRbxCloud ran and there was no metadata disks found (LABEL=CLOUDMD), then it would stacktrace. The fix is just to clean up the get_md function a little bit, and the explicitly check for False as a return value in _get_data. --- cloudinit/sources/DataSourceRbxCloud.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index e064c8d6..0b8994bf 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -71,11 +71,13 @@ def gratuitous_arp(items, distro): def get_md(): - rbx_data = None + """Returns False (not found or error) or a dictionary with metadata.""" devices = set( util.find_devs_with('LABEL=CLOUDMD') + util.find_devs_with('LABEL=cloudmd') ) + if not devices: + return False for device in devices: try: rbx_data = util.mount_cb( @@ -84,17 +86,17 @@ def get_md(): mtype=['vfat', 'fat', 'msdosfs'] ) if rbx_data: - break + return rbx_data except OSError as err: if err.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for user " "data", device) - if not rbx_data: - util.logexc(LOG, "Failed to load metadata and userdata") - return False - return rbx_data + + LOG.debug("Did not find RbxCloud data, searched devices: %s", + ",".join(devices)) + return False def generate_network_config(netadps): @@ -223,6 +225,8 @@ class DataSourceRbxCloud(sources.DataSource): is used to perform instance configuration. """ rbx_data = get_md() + if rbx_data is False: + return False self.userdata_raw = rbx_data['userdata'] self.metadata = rbx_data['metadata'] self.gratuitous_arp = rbx_data['gratuitous_arp'] -- cgit v1.2.3 From 39cf5c4e1575561d622c127797ff9d87e2a649bb Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 10 Nov 2020 11:49:27 -0500 Subject: faq.rst: add warning to answer that suggests running `clean` (#661) --- doc/rtd/topics/faq.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index 944cc27f..d08914b5 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -121,6 +121,12 @@ cloud-init: $ sudo cloud-init init --local $ sudo cloud-init init +.. warning:: + + These commands will re-run cloud-init as if this were first boot of a + system: this will, at the very least, cycle SSH host keys and may do + substantially more. Do not run these commands on production systems. + How can I debug my user data? ============================= -- cgit v1.2.3 From db2ff11dea5d884093e4e29393eeac70d3315bbb Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 10 Nov 2020 12:01:29 -0500 Subject: split integration and cloud_tests requirements (#652) The cloud_tests and the integration_tests (via pycloudlib) have conflicting requirements. This commit splits up their requirements files, so we can accurately express the requirements for each. --- cloud-tests-requirements.txt | 28 ++++++++++++++++++++++++++++ integration-requirements.txt | 29 ++--------------------------- tox.ini | 4 +++- 3 files changed, 33 insertions(+), 28 deletions(-) create mode 100644 cloud-tests-requirements.txt diff --git a/cloud-tests-requirements.txt b/cloud-tests-requirements.txt new file mode 100644 index 00000000..b4cd18d5 --- /dev/null +++ b/cloud-tests-requirements.txt @@ -0,0 +1,28 @@ +# PyPI requirements for cloud-init cloud tests +# https://cloudinit.readthedocs.io/en/latest/topics/cloud_tests.html +# +# Note: Changes to this requirements may require updates to +# the packages/pkg-deps.json file as well. +# + +# ec2 backend +boto3==1.14.53 + +# ssh communication +paramiko==2.7.2 +cryptography==3.1 + +# lxd backend +pylxd==2.2.11 + +# finds latest image information +git+https://git.launchpad.net/simplestreams + +# azure backend +azure-storage==0.36.0 +msrestazure==0.6.1 +azure-common==1.1.23 +azure-mgmt-compute==7.0.0 +azure-mgmt-network==5.0.0 +azure-mgmt-resource==4.0.0 +azure-mgmt-storage==6.0.0 diff --git a/integration-requirements.txt b/integration-requirements.txt index 64455c79..7060841c 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,30 +1,5 @@ # PyPI requirements for cloud-init integration testing -# https://cloudinit.readthedocs.io/en/latest/topics/tests.html -# -# Note: Changes to this requirements may require updates to -# the packages/pkg-deps.json file as well. +# https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # +pycloudlib @ git+https://github.com/canonical/pycloudlib.git pytest -git+https://github.com/canonical/pycloudlib.git - -# ec2 backend -boto3==1.14.53 - -# ssh communication -paramiko==2.7.2 -cryptography==3.1 - -# lxd backend -pylxd==2.2.11 - -# finds latest image information -git+https://git.launchpad.net/simplestreams - -# azure backend -azure-storage==0.36.0 -msrestazure==0.6.1 -azure-common==1.1.23 -azure-mgmt-compute==7.0.0 -azure-mgmt-network==5.0.0 -azure-mgmt-resource==4.0.0 -azure-mgmt-storage==6.0.0 diff --git a/tox.ini b/tox.ini index 816e6e8e..32174dee 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps = pylint==2.6.0 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt + -r{toxinidir}/cloud-tests-requirements.txt -r{toxinidir}/integration-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit tests tools} @@ -128,6 +129,7 @@ deps = pylint # test-requirements -r{toxinidir}/test-requirements.txt + -r{toxinidir}/cloud-tests-requirements.txt -r{toxinidir}/integration-requirements.txt [testenv:citest] @@ -135,7 +137,7 @@ basepython = python3 commands = {envpython} -m tests.cloud_tests {posargs} passenv = HOME TRAVIS deps = - -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/cloud-tests-requirements.txt [testenv:integration-tests] basepython = python3 -- cgit v1.2.3 From 38ba6b3087829a30c9888e60f5f6c6812ae5cfc7 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 13 Nov 2020 11:56:53 -0500 Subject: cloud_tests: add hirsute release definition (#662) --- tests/cloud_tests/releases.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index e76a3d35..6249efc5 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -133,6 +133,22 @@ features: releases: # UBUNTU ================================================================= + hirsute: + # EOL: Jan 2022 + default: + enabled: true + release: hirsute + version: "21.04" + os: ubuntu + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: hirsute + setup_overrides: null + override_templates: false groovy: # EOL: Jul 2021 default: -- cgit v1.2.3 From 8a9ee02dce8485961b0eeb1930e637a88b03b0ad Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 13 Nov 2020 17:34:44 -0500 Subject: DataSourceOpenNebula: exclude SRANDOM from context output (#665) This is a new builtin variable that appeared in Ubuntu in 5.1~rc2-1ubuntu1 and started causing daily build failures. --- cloudinit/sources/DataSourceOpenNebula.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 45481938..730ec586 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -350,7 +350,8 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, # exclude vars in bash that change on their own or that we used excluded = ( "EPOCHREALTIME", "EPOCHSECONDS", "RANDOM", "LINENO", "SECONDS", "_", - "__v") + "SRANDOM", "__v", + ) preset = {} ret = {} target = None -- cgit v1.2.3 From eeef783b0322d99840f39a58de052772afefe822 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 16 Nov 2020 07:58:26 -0600 Subject: Pin pycloudlib to a working commit (#666) --- integration-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-requirements.txt b/integration-requirements.txt index 7060841c..61d2e504 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -pycloudlib @ git+https://github.com/canonical/pycloudlib.git +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@e6b2b3732a2a17d48bdf1167f56eb14576215d3c pytest -- cgit v1.2.3 From a925b5a0ca4aa3e63b084c0f6664fe815c2c9db0 Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Tue, 17 Nov 2020 20:54:14 +0100 Subject: add --no-tty option to gpg (#669) Make sure that gpg works even if the instance has no /dev/tty. This has been observed on Debian. LP: #1813396 --- cloudinit/gpg.py | 2 +- cloudinit/tests/test_gpg.py | 3 ++- tools/.github-cla-signers | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index be0ca0ea..3780326c 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -42,7 +42,7 @@ def recv_key(key, keyserver, retries=(1, 1)): @param retries: an iterable of sleep lengths for retries. Use None to indicate no retries.""" LOG.debug("Importing key '%s' from keyserver '%s'", key, keyserver) - cmd = ["gpg", "--keyserver=%s" % keyserver, "--recv-keys", key] + cmd = ["gpg", "--no-tty", "--keyserver=%s" % keyserver, "--recv-keys", key] if retries is None: retries = [] trynum = 0 diff --git a/cloudinit/tests/test_gpg.py b/cloudinit/tests/test_gpg.py index f96f5372..311dfad6 100644 --- a/cloudinit/tests/test_gpg.py +++ b/cloudinit/tests/test_gpg.py @@ -49,6 +49,7 @@ class TestReceiveKeys(CiTestCase): m_subp.return_value = ('', '') gpg.recv_key(key, keyserver, retries=retries) m_subp.assert_called_once_with( - ['gpg', '--keyserver=%s' % keyserver, '--recv-keys', key], + ['gpg', '--no-tty', + '--keyserver=%s' % keyserver, '--recv-keys', key], capture=True) m_sleep.assert_not_called() diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 2090dc2a..b2464768 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -21,6 +21,7 @@ matthewruffell nishigori omBratteng onitake +riedel slyon smoser sshedi -- cgit v1.2.3 From 4f2da1cc1d24cbc47025cc8613c0d3ec287a20f9 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 17 Nov 2020 16:37:29 -0500 Subject: introduce an upgrade framework and related testing (#659) This commit does the following: * introduces the `cloudinit.persistence` module, containing `CloudInitPickleMixin` which provides lightweight versioning of objects' pickled representations (and associated testing) * introduces a basic upgrade testing framework (in `cloudinit.tests.test_upgrade`) which unpickles pickles from previous versions of cloud-init (stored in `tests/data/old_pickles`) and tests invariants that the current cloud-init codebase expects * uses the versioning framework to address an upgrade issue where `Distro.networking` could get into an unexpected state, and uses the upgrade testing framework to confirm that the issue is addressed --- cloudinit/distros/__init__.py | 17 ++- cloudinit/persistence.py | 67 ++++++++++++ cloudinit/tests/test_persistence.py | 118 +++++++++++++++++++++ cloudinit/tests/test_upgrade.py | 45 ++++++++ .../focal-20.1-10-g71af48df-0ubuntu5.pkl | Bin 0 -> 7135 bytes .../focal-20.3-2-g371b392c-0ubuntu1~20.04.1.pkl | Bin 0 -> 7215 bytes 6 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 cloudinit/persistence.py create mode 100644 cloudinit/tests/test_persistence.py create mode 100644 cloudinit/tests/test_upgrade.py create mode 100644 tests/data/old_pickles/focal-20.1-10-g71af48df-0ubuntu5.pkl create mode 100644 tests/data/old_pickles/focal-20.3-2-g371b392c-0ubuntu1~20.04.1.pkl diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index fac8cf67..1e118472 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -23,6 +23,7 @@ from cloudinit import net from cloudinit.net import eni from cloudinit.net import network_state from cloudinit.net import renderers +from cloudinit import persistence from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import subp @@ -62,7 +63,7 @@ PREFERRED_NTP_CLIENTS = ['chrony', 'systemd-timesyncd', 'ntp', 'ntpdate'] LDH_ASCII_CHARS = string.ascii_letters + string.digits + "-" -class Distro(metaclass=abc.ABCMeta): +class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -77,12 +78,26 @@ class Distro(metaclass=abc.ABCMeta): # subclasses shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} + _ci_pkl_version = 1 + def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name self.networking = self.networking_cls() + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform deserialization fixes for Distro.""" + if "networking" not in self.__dict__ or not self.networking.__dict__: + # This is either a Distro pickle with no networking attribute OR + # this is a Distro pickle with a networking attribute but from + # before ``Networking`` had any state (meaning that + # Networking.__setstate__ will not be called). In either case, we + # want to ensure that `self.networking` is freshly-instantiated: + # either because it isn't present at all, or because it will be + # missing expected instance state otherwise. + self.networking = self.networking_cls() + @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() diff --git a/cloudinit/persistence.py b/cloudinit/persistence.py new file mode 100644 index 00000000..85aa79df --- /dev/null +++ b/cloudinit/persistence.py @@ -0,0 +1,67 @@ +# Copyright (C) 2020 Canonical Ltd. +# +# Author: Daniel Watkins +# +# This file is part of cloud-init. See LICENSE file for license information. + + +class CloudInitPickleMixin: + """Scaffolding for versioning of pickles. + + This class implements ``__getstate__`` and ``__setstate__`` to provide + lightweight versioning of the pickles that are generated for classes which + use it. Versioning is done at the class level. + + The current version of a class's pickle should be set in the class variable + ``_ci_pkl_version``, as an int. If not overriden, it will default to 0. + + On unpickle, the object's state will be restored and then + ``self._unpickle`` is called with the version of the stored pickle as the + only argument: this is where classes should implement any deserialization + fixes they require. (If the stored pickle has no version, 0 is passed.) + """ + + _ci_pkl_version = 0 + + def __getstate__(self): + """Persist instance state, adding a pickle version attribute. + + This adds a ``_ci_pkl_version`` attribute to ``self.__dict__`` and + returns that for serialisation. The attribute is stripped out in + ``__setstate__`` on unpickle. + + The value of ``_ci_pkl_version`` is ``type(self)._ci_pkl_version``. + """ + state = self.__dict__.copy() + state["_ci_pkl_version"] = type(self)._ci_pkl_version + return state + + def __setstate__(self, state: dict) -> None: + """Restore instance state and handle missing attributes on upgrade. + + This will be called when an instance of this class is unpickled; the + previous instance's ``__dict__`` is passed as ``state``. This method + removes the pickle version from the stored state, restores the + remaining state into the current instance, and then calls + ``self._unpickle`` with the version (or 0, if no version is found in + the stored state). + + See https://docs.python.org/3/library/pickle.html#object.__setstate__ + for further background. + """ + version = state.pop("_ci_pkl_version", 0) + self.__dict__.update(state) + self._unpickle(version) + + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform any deserialization fixes required. + + By default, this does nothing. Classes using this mixin should + override this method if they have fixes they need to apply. + + ``ci_pkl_version`` will be the version stored in the pickle for this + object, or 0 if no version is present. + """ + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_persistence.py b/cloudinit/tests/test_persistence.py new file mode 100644 index 00000000..7b64ced9 --- /dev/null +++ b/cloudinit/tests/test_persistence.py @@ -0,0 +1,118 @@ +# Copyright (C) 2020 Canonical Ltd. +# +# Author: Daniel Watkins +# +# This file is part of cloud-init. See LICENSE file for license information. +""" +Tests for cloudinit.persistence. + +Per https://docs.python.org/3/library/pickle.html, only "classes that are +defined at the top level of a module" can be pickled. This means that all of +our ``CloudInitPickleMixin`` subclasses for testing must be defined at +module-level (rather than being defined inline or dynamically in the body of +test methods, as we would do without this constraint). + +``TestPickleMixin.test_subclasses`` iterates over a list of all of these +classes, and tests that they round-trip through a pickle dump/load. As the +interface we're testing is that ``_unpickle`` is called appropriately on +subclasses, our subclasses define their assertions in their ``_unpickle`` +implementation. (This means that the assertions will not be executed if +``_unpickle`` is not called at all; we have +``TestPickleMixin.test_unpickle_called`` to ensure it is called.) + +To avoid manually maintaining a list of classes for parametrization we use a +simple metaclass, ``_Collector``, to gather them up. +""" + +import pickle +from unittest import mock + +import pytest + +from cloudinit.persistence import CloudInitPickleMixin + + +class _Collector(type): + """Any class using this as a metaclass will be stored in test_classes.""" + + test_classes = [] + + def __new__(cls, *args): + new_cls = super().__new__(cls, *args) + _Collector.test_classes.append(new_cls) + return new_cls + + +class InstanceVersionNotUsed(CloudInitPickleMixin, metaclass=_Collector): + """Test that the class version is used over one set in instance state.""" + + _ci_pkl_version = 1 + + def __init__(self): + self._ci_pkl_version = 2 + + def _unpickle(self, ci_pkl_version: int) -> None: + assert 1 == ci_pkl_version + + +class MissingVersionHandled(CloudInitPickleMixin, metaclass=_Collector): + """Test that pickles without ``_ci_pkl_version`` are handled gracefully. + + This is tested by overriding ``__getstate__`` so the dumped pickle of this + class will not have ``_ci_pkl_version`` included. + """ + + def __getstate__(self): + return self.__dict__ + + def _unpickle(self, ci_pkl_version: int) -> None: + assert 0 == ci_pkl_version + + +class OverridenVersionHonored(CloudInitPickleMixin, metaclass=_Collector): + """Test that the subclass's version is used.""" + + _ci_pkl_version = 1 + + def _unpickle(self, ci_pkl_version: int) -> None: + assert 1 == ci_pkl_version + + +class StateIsRestored(CloudInitPickleMixin, metaclass=_Collector): + """Instance state should be restored before ``_unpickle`` is called.""" + + def __init__(self): + self.some_state = "some state" + + def _unpickle(self, ci_pkl_version: int) -> None: + assert "some state" == self.some_state + + +class UnpickleCanBeUnoverriden(CloudInitPickleMixin, metaclass=_Collector): + """Subclasses should not need to override ``_unpickle``.""" + + +class VersionDefaultsToZero(CloudInitPickleMixin, metaclass=_Collector): + """Test that the default version is 0.""" + + def _unpickle(self, ci_pkl_version: int) -> None: + assert 0 == ci_pkl_version + + +class TestPickleMixin: + def test_unpickle_called(self): + """Test that self._unpickle is called on unpickle.""" + with mock.patch.object( + CloudInitPickleMixin, "_unpickle" + ) as m_unpickle: + pickle.loads(pickle.dumps(CloudInitPickleMixin())) + assert 1 == m_unpickle.call_count + + @pytest.mark.parametrize("cls", _Collector.test_classes) + def test_subclasses(self, cls): + """For each collected class, round-trip through pickle dump/load. + + Assertions are implemented in ``cls._unpickle``, and so are evoked as + part of the pickle load. + """ + pickle.loads(pickle.dumps(cls())) diff --git a/cloudinit/tests/test_upgrade.py b/cloudinit/tests/test_upgrade.py new file mode 100644 index 00000000..f79a2536 --- /dev/null +++ b/cloudinit/tests/test_upgrade.py @@ -0,0 +1,45 @@ +# Copyright (C) 2020 Canonical Ltd. +# +# Author: Daniel Watkins +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Upgrade testing for cloud-init. + +This module tests cloud-init's behaviour across upgrades. Specifically, it +specifies a set of invariants that the current codebase expects to be true (as +tests in ``TestUpgrade``) and then checks that these hold true after unpickling +``obj.pkl``s from previous versions of cloud-init; those pickles are stored in +``tests/data/old_pickles/``. +""" + +import operator +import pathlib + +import pytest + +from cloudinit.stages import _pkl_load +from cloudinit.tests.helpers import resourceLocation + + +class TestUpgrade: + @pytest.fixture( + params=pathlib.Path(resourceLocation("old_pickles")).glob("*.pkl"), + scope="class", + ids=operator.attrgetter("name"), + ) + def previous_obj_pkl(self, request): + """Load each pickle to memory once, then run all tests against it. + + Test implementations _must not_ modify the ``previous_obj_pkl`` which + they are passed, as that will affect tests that run after them. + """ + return _pkl_load(str(request.param)) + + def test_networking_set_on_distro(self, previous_obj_pkl): + """We always expect to have ``.networking`` on ``Distro`` objects.""" + assert previous_obj_pkl.distro.networking is not None + + def test_blacklist_drivers_set_on_networking(self, previous_obj_pkl): + """We always expect Networking.blacklist_drivers to be initialised.""" + assert previous_obj_pkl.distro.networking.blacklist_drivers is None diff --git a/tests/data/old_pickles/focal-20.1-10-g71af48df-0ubuntu5.pkl b/tests/data/old_pickles/focal-20.1-10-g71af48df-0ubuntu5.pkl new file mode 100644 index 00000000..358813b4 Binary files /dev/null and b/tests/data/old_pickles/focal-20.1-10-g71af48df-0ubuntu5.pkl differ diff --git a/tests/data/old_pickles/focal-20.3-2-g371b392c-0ubuntu1~20.04.1.pkl b/tests/data/old_pickles/focal-20.3-2-g371b392c-0ubuntu1~20.04.1.pkl new file mode 100644 index 00000000..e26f98d8 Binary files /dev/null and b/tests/data/old_pickles/focal-20.3-2-g371b392c-0ubuntu1~20.04.1.pkl differ -- cgit v1.2.3 From e1bde919923ff1f9d1d197f64ea43b976f034062 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 17 Nov 2020 16:55:55 -0500 Subject: test_persistence: add VersionIsPoppedFromState test (#673) --- cloudinit/tests/test_persistence.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cloudinit/tests/test_persistence.py b/cloudinit/tests/test_persistence.py index 7b64ced9..3e5eaa8d 100644 --- a/cloudinit/tests/test_persistence.py +++ b/cloudinit/tests/test_persistence.py @@ -99,6 +99,16 @@ class VersionDefaultsToZero(CloudInitPickleMixin, metaclass=_Collector): assert 0 == ci_pkl_version +class VersionIsPoppedFromState(CloudInitPickleMixin, metaclass=_Collector): + """Test _ci_pkl_version is popped from state before being restored.""" + + def _unpickle(self, ci_pkl_version: int) -> None: + # `self._ci_pkl_version` returns the type's _ci_pkl_version if it isn't + # in instance state, so we need to explicitly check self.__dict__. + sentinel = mock.sentinel.default + assert self.__dict__.get("_ci_pkl_version", sentinel) == sentinel + + class TestPickleMixin: def test_unpickle_called(self): """Test that self._unpickle is called on unpickle.""" -- cgit v1.2.3 From f680114446a5a20ce88f3d10d966811a774c8e8f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 18 Nov 2020 07:23:44 -0700 Subject: cli: add --system param to allow validating system user-data on a machine (#575) Allow root user to validate the userdata provided to the launched machine using `cloud-init devel schema --system` --- cloudinit/config/schema.py | 41 ++++++++--- doc/rtd/topics/faq.rst | 6 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_handler/test_schema.py | 109 ++++++++++++++++++++-------- 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 8a966aee..456bab2c 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" +from cloudinit.cmd.devel import read_cfg_paths from cloudinit import importer from cloudinit.util import find_modules, load_file @@ -173,7 +174,8 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): def validate_cloudconfig_file(config_path, schema, annotate=False): """Validate cloudconfig file adheres to a specific jsonschema. - @param config_path: Path to the yaml cloud-config file to parse. + @param config_path: Path to the yaml cloud-config file to parse, or None + to default to system userdata from Paths object. @param schema: Dict describing a valid jsonschema to validate against. @param annotate: Boolean set True to print original config file with error annotations on the offending lines. @@ -181,9 +183,24 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): @raises SchemaValidationError containing any of schema_errors encountered. @raises RuntimeError when config_path does not exist. """ - if not os.path.exists(config_path): - raise RuntimeError('Configfile {0} does not exist'.format(config_path)) - content = load_file(config_path, decode=False) + if config_path is None: + # Use system's raw userdata path + if os.getuid() != 0: + raise RuntimeError( + "Unable to read system userdata as non-root user." + " Try using sudo" + ) + paths = read_cfg_paths() + user_data_file = paths.get_ipath_cur("userdata_raw") + content = load_file(user_data_file, decode=False) + else: + if not os.path.exists(config_path): + raise RuntimeError( + 'Configfile {0} does not exist'.format( + config_path + ) + ) + content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = ( ('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( @@ -425,6 +442,8 @@ def get_parser(parser=None): description='Validate cloud-config files or document schema') parser.add_argument('-c', '--config-file', help='Path of the cloud-config yaml file to validate') + parser.add_argument('--system', action='store_true', default=False, + help='Validate the system cloud-config userdata') parser.add_argument('-d', '--docs', nargs='+', help=('Print schema module docs. Choices: all or' ' space-delimited cc_names.')) @@ -435,11 +454,11 @@ def get_parser(parser=None): def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.docs] - if not any(exclusive_args) or all(exclusive_args): - error('Expected either --config-file argument or --docs') + exclusive_args = [args.config_file, args.docs, args.system] + if len([arg for arg in exclusive_args if arg]) != 1: + error('Expected one of --config-file, --system or --docs arguments') full_schema = get_schema() - if args.config_file: + if args.config_file or args.system: try: validate_cloudconfig_file( args.config_file, full_schema, args.annotate) @@ -449,7 +468,11 @@ def handle_schema_args(name, args): except RuntimeError as e: error(str(e)) else: - print("Valid cloud-config file {0}".format(args.config_file)) + if args.config_file is None: + cfg_name = "system userdata" + else: + cfg_name = args.config_file + print("Valid cloud-config:", cfg_name) elif args.docs: schema_ids = [subschema['id'] for subschema in full_schema['allOf']] schema_ids += ['all'] diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index d08914b5..27fabf15 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -141,12 +141,12 @@ that can validate your user data offline. .. _validate-yaml.py: https://github.com/canonical/cloud-init/blob/master/tools/validate-yaml.py -Another option is to run the following on an instance when debugging: +Another option is to run the following on an instance to debug userdata +provided to the system: .. code-block:: shell-session - $ sudo cloud-init query userdata > user-data.yaml - $ cloud-init devel schema -c user-data.yaml --annotate + $ cloud-init devel schema --system --annotate As launching instances in the cloud can cost money and take a bit longer, sometimes it is easier to launch instances locally using Multipass or LXD: diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index dcf0fe5a..74f85959 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -214,7 +214,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self.assertEqual(1, exit_code) # Known whitebox output from schema subcommand self.assertEqual( - 'Expected either --config-file argument or --docs\n', + 'Expected one of --config-file, --system or --docs arguments\n', self.stderr.getvalue()) def test_wb_devel_schema_subcommand_doc_content(self): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 44292571..15aa77bb 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -9,9 +9,9 @@ from cloudinit.util import write_file from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema from copy import copy +import itertools import os import pytest -from io import StringIO from pathlib import Path from textwrap import dedent from yaml import safe_load @@ -400,50 +400,97 @@ class AnnotatedCloudconfigFileTest(CiTestCase): annotated_cloudconfig_file(parsed_config, content, schema_errors)) -class MainTest(CiTestCase): +class TestMain: - def test_main_missing_args(self): + exclusive_combinations = itertools.combinations( + ["--system", "--docs all", "--config-file something"], 2 + ) + + @pytest.mark.parametrize("params", exclusive_combinations) + def test_main_exclusive_args(self, params, capsys): + """Main exits non-zero and error on required exclusive args.""" + params = list(itertools.chain(*[a.split() for a in params])) + with mock.patch('sys.argv', ['mycmd'] + params): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err + + def test_main_missing_args(self, capsys): """Main exits non-zero and reports an error on missing parameters.""" with mock.patch('sys.argv', ['mycmd']): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Expected either --config-file argument or --docs\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err - def test_main_absent_config_file(self): + def test_main_absent_config_file(self, capsys): """Main exits non-zero when config file is absent.""" myargs = ['mycmd', '--annotate', '--config-file', 'NOT_A_FILE'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Configfile NOT_A_FILE does not exist\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + assert 'Configfile NOT_A_FILE does not exist\n' == err - def test_main_prints_docs(self): + def test_main_prints_docs(self, capsys): """When --docs parameter is provided, main generates documentation.""" myargs = ['mycmd', '--docs', 'all'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn('\nNTP\n---\n', m_stdout.getvalue()) - self.assertIn('\nRuncmd\n------\n', m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert '\nNTP\n---\n' in out + assert '\nRuncmd\n------\n' in out - def test_main_validates_config_file(self): + def test_main_validates_config_file(self, tmpdir, capsys): """When --config-file parameter is provided, main validates schema.""" - myyaml = self.tmp_path('my.yaml') - myargs = ['mycmd', '--config-file', myyaml] - write_file(myyaml, b'#cloud-config\nntp:') # shortest ntp schema + myyaml = tmpdir.join('my.yaml') + myargs = ['mycmd', '--config-file', myyaml.strpath] + myyaml.write(b'#cloud-config\nntp:') # shortest ntp schema with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn( - 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: {0}\n'.format(myyaml) == out + + @mock.patch('cloudinit.config.schema.read_cfg_paths') + @mock.patch('cloudinit.config.schema.os.getuid', return_value=0) + def test_main_validates_system_userdata( + self, m_getuid, m_read_cfg_paths, capsys, paths + ): + """When --system is provided, main validates system userdata.""" + m_read_cfg_paths.return_value = paths + ud_file = paths.get_ipath_cur("userdata_raw") + write_file(ud_file, b'#cloud-config\nntp:') + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: system userdata\n' == out + + @mock.patch('cloudinit.config.schema.os.getuid', return_value=1000) + def test_main_system_userdata_requires_root(self, m_getuid, capsys, paths): + """Non-root user can't use --system param""" + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + expected = ( + 'Unable to read system userdata as non-root user. Try using sudo\n' + ) + assert expected == err class CloudTestsIntegrationTest(CiTestCase): -- cgit v1.2.3 From cd752df6154c403e6dccaf5e797c1d4f8396f756 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Nov 2020 09:48:47 -0500 Subject: only run a subset of integration tests in CI (#672) This introduces the "ci" mark, used to indicate a test which should run as part of our CI integration testing run and the integration-tests-ci tox environment, which runs only those tests. Travis has been adjusted to use this tox environment. (All current module tests have been marked with the "ci" mark, but the one bug test that we have has not.) --- .travis.yml | 2 +- .../integration_tests/modules/test_apt_configure_sources_list.py | 1 + tests/integration_tests/modules/test_ntp_servers.py | 1 + tests/integration_tests/modules/test_runcmd.py | 1 + tests/integration_tests/modules/test_seed_random_data.py | 1 + tests/integration_tests/modules/test_set_hostname.py | 1 + tests/integration_tests/modules/test_set_password.py | 2 ++ tests/integration_tests/modules/test_snap.py | 1 + tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py | 1 + tests/integration_tests/modules/test_ssh_generate.py | 1 + tests/integration_tests/modules/test_ssh_import_id.py | 1 + tests/integration_tests/modules/test_ssh_keys_provided.py | 1 + tests/integration_tests/modules/test_timezone.py | 1 + tests/integration_tests/modules/test_users_groups.py | 1 + tests/integration_tests/modules/test_write_files.py | 1 + tox.ini | 8 ++++++++ 16 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 496c1a81..6d25f477 100644 --- a/.travis.yml +++ b/.travis.yml @@ -199,7 +199,7 @@ matrix: fi # Use sudo to get a new shell where we're in the sbuild group - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' - - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' & + - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' & - | SECONDS=0 while [ -e /proc/$! ]; do diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py index d64b3956..d2bcc61a 100644 --- a/tests/integration_tests/modules/test_apt_configure_sources_list.py +++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py @@ -39,6 +39,7 @@ EXPECTED_REGEXES = [ ] +@pytest.mark.ci class TestAptConfigureSourcesList: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py index 4cad8926..e72389c1 100644 --- a/tests/integration_tests/modules/test_ntp_servers.py +++ b/tests/integration_tests/modules/test_ntp_servers.py @@ -22,6 +22,7 @@ ntp: EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"] +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestNtpServers: diff --git a/tests/integration_tests/modules/test_runcmd.py b/tests/integration_tests/modules/test_runcmd.py index eabe778d..50d1851e 100644 --- a/tests/integration_tests/modules/test_runcmd.py +++ b/tests/integration_tests/modules/test_runcmd.py @@ -16,6 +16,7 @@ runcmd: """ +@pytest.mark.ci class TestRuncmd: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py index db3d2193..b365fa98 100644 --- a/tests/integration_tests/modules/test_seed_random_data.py +++ b/tests/integration_tests/modules/test_seed_random_data.py @@ -19,6 +19,7 @@ random_seed: """ +@pytest.mark.ci class TestSeedRandomData: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_set_hostname.py b/tests/integration_tests/modules/test_set_hostname.py index ff46feb9..2bfa403d 100644 --- a/tests/integration_tests/modules/test_set_hostname.py +++ b/tests/integration_tests/modules/test_set_hostname.py @@ -25,6 +25,7 @@ fqdn: cloudinit2.i9n.cloud-init.io """ +@pytest.mark.ci class TestHostname: @pytest.mark.user_data(USER_DATA_HOSTNAME) diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py index ae6fdefc..b13f76fb 100644 --- a/tests/integration_tests/modules/test_set_password.py +++ b/tests/integration_tests/modules/test_set_password.py @@ -139,11 +139,13 @@ class Mixin: assert "PasswordAuthentication yes" in sshd_config.splitlines() +@pytest.mark.ci @pytest.mark.user_data(LIST_USER_DATA) class TestPasswordList(Mixin): """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass.""" +@pytest.mark.ci @pytest.mark.user_data(STRING_USER_DATA) class TestPasswordListString(Mixin): """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass.""" diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py index d78a0b1e..b626f6b0 100644 --- a/tests/integration_tests/modules/test_snap.py +++ b/tests/integration_tests/modules/test_snap.py @@ -19,6 +19,7 @@ snap: """ +@pytest.mark.ci class TestSnap: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py index e88d9a02..b9b0d85e 100644 --- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -28,6 +28,7 @@ ssh_authorized_keys: """ # noqa +@pytest.mark.ci class TestSshAuthkeyFingerprints: @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_DISABLE) diff --git a/tests/integration_tests/modules/test_ssh_generate.py b/tests/integration_tests/modules/test_ssh_generate.py index 8c60fb87..60c36982 100644 --- a/tests/integration_tests/modules/test_ssh_generate.py +++ b/tests/integration_tests/modules/test_ssh_generate.py @@ -20,6 +20,7 @@ authkey_hash: sha512 """ +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestSshKeysGenerate: diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py index 2f2ac92c..45d37d6c 100644 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ b/tests/integration_tests/modules/test_ssh_import_id.py @@ -17,6 +17,7 @@ ssh_import_id: """ +@pytest.mark.ci class TestSshImportId: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py index 4699518d..dc6d2fc1 100644 --- a/tests/integration_tests/modules/test_ssh_keys_provided.py +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -78,6 +78,7 @@ ssh_keys: """ # noqa +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestSshKeysProvided: diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py index 6080d79e..111d53f7 100644 --- a/tests/integration_tests/modules/test_timezone.py +++ b/tests/integration_tests/modules/test_timezone.py @@ -15,6 +15,7 @@ timezone: US/Aleutian """ +@pytest.mark.ci class TestTimezone: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index b1fa8c22..6a085a8f 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -38,6 +38,7 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ """ +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestUsersGroups: @pytest.mark.parametrize( diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py index d7032a0c..15832ae3 100644 --- a/tests/integration_tests/modules/test_write_files.py +++ b/tests/integration_tests/modules/test_write_files.py @@ -44,6 +44,7 @@ write_files: """.format(B64_CONTENT.decode("ascii")) +@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestWriteFiles: diff --git a/tox.ini b/tox.ini index 32174dee..4320ab87 100644 --- a/tox.ini +++ b/tox.ini @@ -146,6 +146,13 @@ passenv = CLOUD_INIT_* deps = -r{toxinidir}/integration-requirements.txt +[testenv:integration-tests-ci] +commands = {[testenv:integration-tests]commands} +passenv = {[testenv:integration-tests]passenv} +deps = {[testenv:integration-tests]deps} +setenv = + PYTEST_ADDOPTS="-k ci" + [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped testpaths = cloudinit tests/unittests @@ -153,6 +160,7 @@ addopts = --strict markers = allow_subp_for: allow subp usage for the given commands (disable_subp_usage) allow_all_subp: allow all subp usage (disable_subp_usage) + ci: run this integration test as part of CI test runs ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures ec2: test will only run on EC2 platform gce: test will only run on GCE platform -- cgit v1.2.3 From 96d21dfbee308cd8fe00809184f78da9231ece4a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Nov 2020 11:11:58 -0500 Subject: test_persistence: simplify VersionIsPoppedFromState (#674) --- cloudinit/tests/test_persistence.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/tests/test_persistence.py b/cloudinit/tests/test_persistence.py index 3e5eaa8d..ec1152a9 100644 --- a/cloudinit/tests/test_persistence.py +++ b/cloudinit/tests/test_persistence.py @@ -105,8 +105,7 @@ class VersionIsPoppedFromState(CloudInitPickleMixin, metaclass=_Collector): def _unpickle(self, ci_pkl_version: int) -> None: # `self._ci_pkl_version` returns the type's _ci_pkl_version if it isn't # in instance state, so we need to explicitly check self.__dict__. - sentinel = mock.sentinel.default - assert self.__dict__.get("_ci_pkl_version", sentinel) == sentinel + assert "_ci_pkl_version" not in self.__dict__ class TestPickleMixin: -- cgit v1.2.3 From d807df288f8cef29ca74f0b00c326b084e825782 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 18 Nov 2020 09:34:04 -0800 Subject: DataSourceAzure: send failure signal on Azure datasource failure (#594) On systems where the Azure datasource is a viable platform for crawling metadata, cloud-init occasionally encounters fatal irrecoverable errors during the crawling of the Azure datasource. When this happens, cloud-init crashes, and Azure VM provisioning would fail. However, instead of failing immediately, the user will continue seeing provisioning for a long time until it times out with "OS Provisioning Timed Out" message. In these situations, cloud-init should report failure to the Azure datasource endpoint indicating provisioning failure. The user will immediately see provisioning terminate, giving them a much better failure experience instead of pointlessly waiting for OS provisioning timeout. --- cloudinit/sources/DataSourceAzure.py | 73 ++- cloudinit/sources/helpers/azure.py | 80 ++- tests/unittests/test_datasource/test_azure.py | 322 ++++++++++-- .../unittests/test_datasource/test_azure_helper.py | 569 ++++++++++++++++++--- 4 files changed, 921 insertions(+), 123 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index fa3e0a2b..ab139b8d 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.reporting import events from cloudinit.sources.helpers.azure import ( + DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE, azure_ds_reporter, azure_ds_telemetry_reporter, get_metadata_from_fabric, @@ -38,7 +39,8 @@ from cloudinit.sources.helpers.azure import ( EphemeralDHCPv4WithReporting, is_byte_swapped, dhcp_log_cb, - push_log_to_kvp) + push_log_to_kvp, + report_failure_to_fabric) LOG = logging.getLogger(__name__) @@ -508,8 +510,9 @@ class DataSourceAzure(sources.DataSource): if perform_reprovision: LOG.info("Reporting ready to Azure after getting ReprovisionData") - use_cached_ephemeral = (net.is_up(self.fallback_interface) and - getattr(self, '_ephemeral_dhcp_ctx', None)) + use_cached_ephemeral = ( + self.distro.networking.is_up(self.fallback_interface) and + getattr(self, '_ephemeral_dhcp_ctx', None)) if use_cached_ephemeral: self._report_ready(lease=self._ephemeral_dhcp_ctx.lease) self._ephemeral_dhcp_ctx.clean_network() # Teardown ephemeral @@ -560,9 +563,14 @@ class DataSourceAzure(sources.DataSource): logfunc=LOG.debug, msg='Crawl of metadata service', func=self.crawl_metadata ) - except sources.InvalidMetaDataException as e: - LOG.warning('Could not crawl Azure metadata: %s', e) + except Exception as e: + report_diagnostic_event( + 'Could not crawl Azure metadata: %s' % e, + logger_func=LOG.error) + self._report_failure( + description=DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE) return False + if (self.distro and self.distro.name == 'ubuntu' and self.ds_cfg.get('apply_network_config')): maybe_remove_ubuntu_network_config_scripts() @@ -785,6 +793,61 @@ class DataSourceAzure(sources.DataSource): return return_val @azure_ds_telemetry_reporter + def _report_failure(self, description=None) -> bool: + """Tells the Azure fabric that provisioning has failed. + + @param description: A description of the error encountered. + @return: The success status of sending the failure signal. + """ + unknown_245_key = 'unknown-245' + + try: + if (self.distro.networking.is_up(self.fallback_interface) and + getattr(self, '_ephemeral_dhcp_ctx', None) and + getattr(self._ephemeral_dhcp_ctx, 'lease', None) and + unknown_245_key in self._ephemeral_dhcp_ctx.lease): + report_diagnostic_event( + 'Using cached ephemeral dhcp context ' + 'to report failure to Azure', logger_func=LOG.debug) + report_failure_to_fabric( + dhcp_opts=self._ephemeral_dhcp_ctx.lease[unknown_245_key], + description=description) + self._ephemeral_dhcp_ctx.clean_network() # Teardown ephemeral + return True + except Exception as e: + report_diagnostic_event( + 'Failed to report failure using ' + 'cached ephemeral dhcp context: %s' % e, + logger_func=LOG.error) + + try: + report_diagnostic_event( + 'Using new ephemeral dhcp to report failure to Azure', + logger_func=LOG.debug) + with EphemeralDHCPv4WithReporting(azure_ds_reporter) as lease: + report_failure_to_fabric( + dhcp_opts=lease[unknown_245_key], + description=description) + return True + except Exception as e: + report_diagnostic_event( + 'Failed to report failure using new ephemeral dhcp: %s' % e, + logger_func=LOG.debug) + + try: + report_diagnostic_event( + 'Using fallback lease to report failure to Azure') + report_failure_to_fabric( + fallback_lease_file=self.dhclient_lease_file, + description=description) + return True + except Exception as e: + report_diagnostic_event( + 'Failed to report failure using fallback lease: %s' % e, + logger_func=LOG.debug) + + return False + def _report_ready(self, lease: dict) -> bool: """Tells the fabric provisioning has completed. diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 4071a50e..951c7a10 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -17,6 +17,7 @@ from cloudinit import stages from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree +from xml.sax.saxutils import escape from cloudinit import subp from cloudinit import url_helper @@ -50,6 +51,11 @@ azure_ds_reporter = events.ReportEventStack( description="initialize reporter for azure ds", reporting_enabled=True) +DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE = ( + 'The VM encountered an error during deployment. ' + 'Please visit https://aka.ms/linuxprovisioningerror ' + 'for more information on remediation.') + def azure_ds_telemetry_reporter(func): def impl(*args, **kwargs): @@ -379,12 +385,20 @@ class OpenSSLManager: def __init__(self): self.tmpdir = temp_utils.mkdtemp() - self.certificate = None + self._certificate = None self.generate_certificate() def clean_up(self): util.del_dir(self.tmpdir) + @property + def certificate(self): + return self._certificate + + @certificate.setter + def certificate(self, value): + self._certificate = value + @azure_ds_telemetry_reporter def generate_certificate(self): LOG.debug('Generating certificate for communication with fabric...') @@ -507,6 +521,10 @@ class GoalStateHealthReporter: ''') PROVISIONING_SUCCESS_STATUS = 'Ready' + PROVISIONING_NOT_READY_STATUS = 'NotReady' + PROVISIONING_FAILURE_SUBSTATUS = 'ProvisioningFailed' + + HEALTH_REPORT_DESCRIPTION_TRIM_LEN = 512 def __init__( self, goal_state: GoalState, @@ -545,19 +563,39 @@ class GoalStateHealthReporter: LOG.info('Reported ready to Azure fabric.') + @azure_ds_telemetry_reporter + def send_failure_signal(self, description: str) -> None: + document = self.build_report( + incarnation=self._goal_state.incarnation, + container_id=self._goal_state.container_id, + instance_id=self._goal_state.instance_id, + status=self.PROVISIONING_NOT_READY_STATUS, + substatus=self.PROVISIONING_FAILURE_SUBSTATUS, + description=description) + try: + self._post_health_report(document=document) + except Exception as e: + msg = "exception while reporting failure: %s" % e + report_diagnostic_event(msg, logger_func=LOG.error) + raise + + LOG.warning('Reported failure to Azure fabric.') + def build_report( self, incarnation: str, container_id: str, instance_id: str, status: str, substatus=None, description=None) -> str: health_detail = '' if substatus is not None: health_detail = self.HEALTH_DETAIL_SUBSECTION_XML_TEMPLATE.format( - health_substatus=substatus, health_description=description) + health_substatus=escape(substatus), + health_description=escape( + description[:self.HEALTH_REPORT_DESCRIPTION_TRIM_LEN])) health_report = self.HEALTH_REPORT_XML_TEMPLATE.format( - incarnation=incarnation, - container_id=container_id, - instance_id=instance_id, - health_status=status, + incarnation=escape(str(incarnation)), + container_id=escape(container_id), + instance_id=escape(instance_id), + health_status=escape(status), health_detail_subsection=health_detail) return health_report @@ -797,6 +835,20 @@ class WALinuxAgentShim: health_reporter.send_ready_signal() return {'public-keys': ssh_keys} + @azure_ds_telemetry_reporter + def register_with_azure_and_report_failure(self, description: str) -> None: + """Gets the VM's GoalState from Azure, uses the GoalState information + to report failure/send provisioning failure signal to Azure. + + @param: user visible error description of provisioning failure. + """ + if self.azure_endpoint_client is None: + self.azure_endpoint_client = AzureEndpointHttpClient(None) + goal_state = self._fetch_goal_state_from_azure(need_certificate=False) + health_reporter = GoalStateHealthReporter( + goal_state, self.azure_endpoint_client, self.endpoint) + health_reporter.send_failure_signal(description=description) + @azure_ds_telemetry_reporter def _fetch_goal_state_from_azure( self, @@ -804,6 +856,7 @@ class WALinuxAgentShim: """Fetches the GoalState XML from the Azure endpoint, parses the XML, and returns a GoalState object. + @param need_certificate: switch to know if certificates is needed. @return: GoalState object representing the GoalState XML """ unparsed_goal_state_xml = self._get_raw_goal_state_xml_from_azure() @@ -844,6 +897,7 @@ class WALinuxAgentShim: """Parses a GoalState XML string and returns a GoalState object. @param unparsed_goal_state_xml: GoalState XML string + @param need_certificate: switch to know if certificates is needed. @return: GoalState object representing the GoalState XML """ try: @@ -942,6 +996,20 @@ def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, shim.clean_up() +@azure_ds_telemetry_reporter +def report_failure_to_fabric(fallback_lease_file=None, dhcp_opts=None, + description=None): + shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file, + dhcp_options=dhcp_opts) + if not description: + description = DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE + try: + shim.register_with_azure_and_report_failure( + description=description) + finally: + shim.clean_up() + + def dhcp_log_cb(out, err): report_diagnostic_event( "dhclient output stream: %s" % out, logger_func=LOG.debug) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 433fbc66..d9752ab7 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -461,6 +461,8 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): class TestAzureDataSource(CiTestCase): + with_logs = True + def setUp(self): super(TestAzureDataSource, self).setUp() self.tmp = self.tmp_dir() @@ -549,9 +551,12 @@ scbus-1 on xpt0 bus 0 dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d - self.get_metadata_from_fabric = mock.MagicMock(return_value={ - 'public-keys': [], - }) + self.m_is_platform_viable = mock.MagicMock(autospec=True) + self.m_get_metadata_from_fabric = mock.MagicMock( + return_value={'public-keys': []}) + self.m_report_failure_to_fabric = mock.MagicMock(autospec=True) + self.m_ephemeral_dhcpv4 = mock.MagicMock() + self.m_ephemeral_dhcpv4_with_reporting = mock.MagicMock() self.instance_id = 'D0DF4C54-4ECB-4A4B-9954-5BDF3ED5C3B8' @@ -568,7 +573,17 @@ scbus-1 on xpt0 bus 0 (dsaz, 'perform_hostname_bounce', mock.MagicMock()), (dsaz, 'get_hostname', mock.MagicMock()), (dsaz, 'set_hostname', mock.MagicMock()), - (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), + (dsaz, '_is_platform_viable', + self.m_is_platform_viable), + (dsaz, 'get_metadata_from_fabric', + self.m_get_metadata_from_fabric), + (dsaz, 'report_failure_to_fabric', + self.m_report_failure_to_fabric), + (dsaz, 'EphemeralDHCPv4', self.m_ephemeral_dhcpv4), + (dsaz, 'EphemeralDHCPv4WithReporting', + self.m_ephemeral_dhcpv4_with_reporting), + (dsaz, 'get_boot_telemetry', mock.MagicMock()), + (dsaz, 'get_system_info', mock.MagicMock()), (dsaz.subp, 'which', lambda x: True), (dsaz.dmi, 'read_dmi_data', mock.MagicMock( side_effect=_dmi_mocks)), @@ -632,15 +647,87 @@ scbus-1 on xpt0 bus 0 dev = ds.get_resource_disk_on_freebsd(1) self.assertEqual("da1", dev) - @mock.patch(MOCKPATH + '_is_platform_viable') - def test_call_is_platform_viable_seed(self, m_is_platform_viable): + def test_not_is_platform_viable_seed_should_return_no_datasource(self): """Check seed_dir using _is_platform_viable and return False.""" # Return a non-matching asset tag value - m_is_platform_viable.return_value = False - dsrc = dsaz.DataSourceAzure( - {}, distro=mock.Mock(), paths=self.paths) - self.assertFalse(dsrc.get_data()) - m_is_platform_viable.assert_called_with(dsrc.seed_dir) + data = {} + dsrc = self._get_ds(data) + self.m_is_platform_viable.return_value = False + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc, '_report_failure') as m_report_failure: + ret = dsrc.get_data() + self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + self.assertFalse(ret) + self.assertNotIn('agent_invoked', data) + # Assert that for non viable platforms, + # there is no communication with the Azure datasource. + self.assertEqual( + 0, + m_crawl_metadata.call_count) + self.assertEqual( + 0, + m_report_failure.call_count) + + def test_platform_viable_but_no_devs_should_return_no_datasource(self): + """For platforms where the Azure platform is viable + (which is indicated by the matching asset tag), + the absence of any devs at all (devs == candidate sources + for crawling Azure datasource) is NOT expected. + Report failure to Azure as this is an unexpected fatal error. + """ + data = {} + dsrc = self._get_ds(data) + with mock.patch.object(dsrc, '_report_failure') as m_report_failure: + self.m_is_platform_viable.return_value = True + ret = dsrc.get_data() + self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + self.assertFalse(ret) + self.assertNotIn('agent_invoked', data) + self.assertEqual( + 1, + m_report_failure.call_count) + + def test_crawl_metadata_exception_returns_no_datasource(self): + data = {} + dsrc = self._get_ds(data) + self.m_is_platform_viable.return_value = True + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata: + m_crawl_metadata.side_effect = Exception + ret = dsrc.get_data() + self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + self.assertEqual( + 1, + m_crawl_metadata.call_count) + self.assertFalse(ret) + self.assertNotIn('agent_invoked', data) + + def test_crawl_metadata_exception_should_report_failure_with_msg(self): + data = {} + dsrc = self._get_ds(data) + self.m_is_platform_viable.return_value = True + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc, '_report_failure') as m_report_failure: + m_crawl_metadata.side_effect = Exception + dsrc.get_data() + self.assertEqual( + 1, + m_crawl_metadata.call_count) + m_report_failure.assert_called_once_with( + description=dsaz.DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE) + + def test_crawl_metadata_exc_should_log_could_not_crawl_msg(self): + data = {} + dsrc = self._get_ds(data) + self.m_is_platform_viable.return_value = True + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata: + m_crawl_metadata.side_effect = Exception + dsrc.get_data() + self.assertEqual( + 1, + m_crawl_metadata.call_count) + self.assertIn( + "Could not crawl Azure metadata", + self.logs.getvalue()) def test_basic_seed_dir(self): odata = {'HostName': "myhost", 'UserName': "myuser"} @@ -761,7 +848,7 @@ scbus-1 on xpt0 bus 0 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') def test_crawl_metadata_on_reprovision_reports_ready( - self, poll_imds_func, report_ready_func, m_write, m_dhcp + self, poll_imds_func, m_report_ready, m_write, m_dhcp ): """If reprovisioning, report ready at the end""" ovfenv = construct_valid_ovf_env( @@ -775,18 +862,16 @@ scbus-1 on xpt0 bus 0 dsrc = self._get_ds(data) poll_imds_func.return_value = ovfenv dsrc.crawl_metadata() - self.assertEqual(1, report_ready_func.call_count) + self.assertEqual(1, m_report_ready.call_count) @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') @mock.patch('cloudinit.sources.helpers.netlink.' 'wait_for_media_disconnect_connect') @mock.patch( 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') - @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') - @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('cloudinit.sources.DataSourceAzure.readurl') def test_crawl_metadata_on_reprovision_reports_ready_using_lease( - self, m_readurl, m_dhcp, m_net, report_ready_func, + self, m_readurl, m_report_ready, m_media_switch, m_write ): """If reprovisioning, report ready using the obtained lease""" @@ -800,20 +885,30 @@ scbus-1 on xpt0 bus 0 } dsrc = self._get_ds(data) - lease = { - 'interface': 'eth9', 'fixed-address': '192.168.2.9', - 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', - 'unknown-245': '624c3620'} - m_dhcp.return_value = [lease] - m_media_switch.return_value = None + with mock.patch.object(dsrc.distro.networking, 'is_up') \ + as m_dsrc_distro_networking_is_up: - reprovision_ovfenv = construct_valid_ovf_env() - m_readurl.return_value = url_helper.StringResponse( - reprovision_ovfenv.encode('utf-8')) + # For this mock, net should not be up, + # so that cached ephemeral won't be used. + # This is so that a NEW ephemeral dhcp lease will be discovered + # and used instead. + m_dsrc_distro_networking_is_up.return_value = False - dsrc.crawl_metadata() - self.assertEqual(2, report_ready_func.call_count) - report_ready_func.assert_called_with(lease=lease) + lease = { + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'} + self.m_ephemeral_dhcpv4_with_reporting.return_value \ + .__enter__.return_value = lease + m_media_switch.return_value = None + + reprovision_ovfenv = construct_valid_ovf_env() + m_readurl.return_value = url_helper.StringResponse( + reprovision_ovfenv.encode('utf-8')) + + dsrc.crawl_metadata() + self.assertEqual(2, m_report_ready.call_count) + m_report_ready.assert_called_with(lease=lease) def test_waagent_d_has_0700_perms(self): # we expect /var/lib/waagent to be created 0700 @@ -971,7 +1066,7 @@ scbus-1 on xpt0 bus 0 dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) - self.assertTrue('default_user' in dsrc.cfg['system_info']) + self.assertIn('default_user', dsrc.cfg['system_info']) defuser = dsrc.cfg['system_info']['default_user'] # default user should be updated username and should not be locked. @@ -993,7 +1088,7 @@ scbus-1 on xpt0 bus 0 dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) - self.assertTrue('default_user' in dsrc.cfg['system_info']) + self.assertIn('default_user', dsrc.cfg['system_info']) defuser = dsrc.cfg['system_info']['default_user'] # default user should be updated username and should not be locked. @@ -1021,14 +1116,6 @@ scbus-1 on xpt0 bus 0 self.assertTrue(ret) self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8')) - def test_no_datasource_expected(self): - # no source should be found if no seed_dir and no devs - data = {} - dsrc = self._get_ds({}) - ret = dsrc.get_data() - self.assertFalse(ret) - self.assertFalse('agent_invoked' in data) - def test_cfg_has_pubkeys_fingerprint(self): odata = {'HostName': "myhost", 'UserName': "myuser"} mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] @@ -1171,21 +1258,168 @@ scbus-1 on xpt0 bus 0 self): dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) dsrc.ds_cfg['agent_command'] = '__builtin__' - self.get_metadata_from_fabric.side_effect = Exception + self.m_get_metadata_from_fabric.side_effect = Exception self.assertFalse(dsrc._report_ready(lease=mock.MagicMock())) + def test_dsaz_report_failure_returns_true_when_report_succeeds(self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + self.assertTrue(dsrc._report_failure()) + self.assertEqual( + 1, + self.m_report_failure_to_fabric.call_count) + + def test_dsaz_report_failure_returns_false_and_does_not_propagate_exc( + self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc, '_ephemeral_dhcp_ctx') \ + as m_ephemeral_dhcp_ctx, \ + mock.patch.object(dsrc.distro.networking, 'is_up') \ + as m_dsrc_distro_networking_is_up: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + # setup mocks to allow using cached ephemeral dhcp lease + m_dsrc_distro_networking_is_up.return_value = True + test_lease_dhcp_option_245 = 'test_lease_dhcp_option_245' + test_lease = {'unknown-245': test_lease_dhcp_option_245} + m_ephemeral_dhcp_ctx.lease = test_lease + + # We expect 3 calls to report_failure_to_fabric, + # because we try 3 different methods of calling report failure. + # The different methods are attempted in the following order: + # 1. Using cached ephemeral dhcp context to report failure to Azure + # 2. Using new ephemeral dhcp to report failure to Azure + # 3. Using fallback lease to report failure to Azure + self.m_report_failure_to_fabric.side_effect = Exception + self.assertFalse(dsrc._report_failure()) + self.assertEqual( + 3, + self.m_report_failure_to_fabric.call_count) + + def test_dsaz_report_failure_description_msg(self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + test_msg = 'Test report failure description message' + self.assertTrue(dsrc._report_failure(description=test_msg)) + self.m_report_failure_to_fabric.assert_called_once_with( + dhcp_opts=mock.ANY, description=test_msg) + + def test_dsaz_report_failure_no_description_msg(self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata: + m_crawl_metadata.side_effect = Exception + + self.assertTrue(dsrc._report_failure()) # no description msg + self.m_report_failure_to_fabric.assert_called_once_with( + dhcp_opts=mock.ANY, description=None) + + def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc, '_ephemeral_dhcp_ctx') \ + as m_ephemeral_dhcp_ctx, \ + mock.patch.object(dsrc.distro.networking, 'is_up') \ + as m_dsrc_distro_networking_is_up: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + # setup mocks to allow using cached ephemeral dhcp lease + m_dsrc_distro_networking_is_up.return_value = True + test_lease_dhcp_option_245 = 'test_lease_dhcp_option_245' + test_lease = {'unknown-245': test_lease_dhcp_option_245} + m_ephemeral_dhcp_ctx.lease = test_lease + + self.assertTrue(dsrc._report_failure()) + + # ensure called with cached ephemeral dhcp lease option 245 + self.m_report_failure_to_fabric.assert_called_once_with( + description=mock.ANY, dhcp_opts=test_lease_dhcp_option_245) + + # ensure cached ephemeral is cleaned + self.assertEqual( + 1, + m_ephemeral_dhcp_ctx.clean_network.call_count) + + def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc.distro.networking, 'is_up') \ + as m_dsrc_distro_networking_is_up: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + # net is not up and cannot use cached ephemeral dhcp + m_dsrc_distro_networking_is_up.return_value = False + # setup ephemeral dhcp lease discovery mock + test_lease_dhcp_option_245 = 'test_lease_dhcp_option_245' + test_lease = {'unknown-245': test_lease_dhcp_option_245} + self.m_ephemeral_dhcpv4_with_reporting.return_value \ + .__enter__.return_value = test_lease + + self.assertTrue(dsrc._report_failure()) + + # ensure called with the newly discovered + # ephemeral dhcp lease option 245 + self.m_report_failure_to_fabric.assert_called_once_with( + description=mock.ANY, dhcp_opts=test_lease_dhcp_option_245) + + def test_dsaz_report_failure_no_net_and_no_dhcp_uses_fallback_lease( + self): + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' + + with mock.patch.object(dsrc, 'crawl_metadata') as m_crawl_metadata, \ + mock.patch.object(dsrc.distro.networking, 'is_up') \ + as m_dsrc_distro_networking_is_up: + # mock crawl metadata failure to cause report failure + m_crawl_metadata.side_effect = Exception + + # net is not up and cannot use cached ephemeral dhcp + m_dsrc_distro_networking_is_up.return_value = False + # ephemeral dhcp discovery failure, + # so cannot use a new ephemeral dhcp + self.m_ephemeral_dhcpv4_with_reporting.return_value \ + .__enter__.side_effect = Exception + + self.assertTrue(dsrc._report_failure()) + + # ensure called with fallback lease + self.m_report_failure_to_fabric.assert_called_once_with( + description=mock.ANY, + fallback_lease_file=dsrc.dhclient_lease_file) + def test_exception_fetching_fabric_data_doesnt_propagate(self): """Errors communicating with fabric should warn, but return True.""" dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) dsrc.ds_cfg['agent_command'] = '__builtin__' - self.get_metadata_from_fabric.side_effect = Exception + self.m_get_metadata_from_fabric.side_effect = Exception ret = self._get_and_setup(dsrc) self.assertTrue(ret) def test_fabric_data_included_in_metadata(self): dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) dsrc.ds_cfg['agent_command'] = '__builtin__' - self.get_metadata_from_fabric.return_value = {'test': 'value'} + self.m_get_metadata_from_fabric.return_value = {'test': 'value'} ret = self._get_and_setup(dsrc) self.assertTrue(ret) self.assertEqual('value', dsrc.metadata['test']) @@ -2053,7 +2287,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): @mock.patch('time.sleep', mock.MagicMock()) @mock.patch(MOCKPATH + 'EphemeralDHCPv4') - def test_poll_imds_re_dhcp_on_timeout(self, m_dhcpv4, report_ready_func, + def test_poll_imds_re_dhcp_on_timeout(self, m_dhcpv4, m_report_ready, m_request, m_media_switch, m_dhcp, m_net): """The poll_imds will retry DHCP on IMDS timeout.""" @@ -2088,8 +2322,8 @@ class TestPreprovisioningPollIMDS(CiTestCase): dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): dsa._poll_imds() - self.assertEqual(report_ready_func.call_count, 1) - report_ready_func.assert_called_with(lease=lease) + self.assertEqual(m_report_ready.call_count, 1) + m_report_ready.assert_called_with(lease=lease) self.assertEqual(3, m_dhcpv4.call_count, 'Expected 3 DHCP calls') self.assertEqual(4, self.tries, 'Expected 4 total reads from IMDS') diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 6e004e34..adf68857 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -5,6 +5,7 @@ import re import unittest from textwrap import dedent from xml.etree import ElementTree +from xml.sax.saxutils import escape, unescape from cloudinit.sources.helpers import azure as azure_helper from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir @@ -70,6 +71,15 @@ HEALTH_REPORT_XML_TEMPLATE = '''\ ''' +HEALTH_DETAIL_SUBSECTION_XML_TEMPLATE = dedent('''\ +
+ {health_substatus} + {health_description} +
+ ''') + +HEALTH_REPORT_DESCRIPTION_TRIM_LEN = 512 + class SentinelException(Exception): pass @@ -461,17 +471,24 @@ class TestOpenSSLManagerActions(CiTestCase): class TestGoalStateHealthReporter(CiTestCase): + maxDiff = None + default_parameters = { 'incarnation': 1634, 'container_id': 'MyContainerId', 'instance_id': 'MyInstanceId' } - test_endpoint = 'TestEndpoint' - test_url = 'http://{0}/machine?comp=health'.format(test_endpoint) + test_azure_endpoint = 'TestEndpoint' + test_health_report_url = 'http://{0}/machine?comp=health'.format( + test_azure_endpoint) test_default_headers = {'Content-Type': 'text/xml; charset=utf-8'} provisioning_success_status = 'Ready' + provisioning_not_ready_status = 'NotReady' + provisioning_failure_substatus = 'ProvisioningFailed' + provisioning_failure_err_description = ( + 'Test error message containing provisioning failure details') def setUp(self): super(TestGoalStateHealthReporter, self).setUp() @@ -496,17 +513,40 @@ class TestGoalStateHealthReporter(CiTestCase): self.GoalState.return_value.incarnation = \ self.default_parameters['incarnation'] + def _text_from_xpath_in_xroot(self, xroot, xpath): + element = xroot.find(xpath) + if element is not None: + return element.text + return None + def _get_formatted_health_report_xml_string(self, **kwargs): return HEALTH_REPORT_XML_TEMPLATE.format(**kwargs) + def _get_formatted_health_detail_subsection_xml_string(self, **kwargs): + return HEALTH_DETAIL_SUBSECTION_XML_TEMPLATE.format(**kwargs) + def _get_report_ready_health_document(self): return self._get_formatted_health_report_xml_string( - incarnation=self.default_parameters['incarnation'], - container_id=self.default_parameters['container_id'], - instance_id=self.default_parameters['instance_id'], - health_status=self.provisioning_success_status, + incarnation=escape(str(self.default_parameters['incarnation'])), + container_id=escape(self.default_parameters['container_id']), + instance_id=escape(self.default_parameters['instance_id']), + health_status=escape(self.provisioning_success_status), health_detail_subsection='') + def _get_report_failure_health_document(self): + health_detail_subsection = \ + self._get_formatted_health_detail_subsection_xml_string( + health_substatus=escape(self.provisioning_failure_substatus), + health_description=escape( + self.provisioning_failure_err_description)) + + return self._get_formatted_health_report_xml_string( + incarnation=escape(str(self.default_parameters['incarnation'])), + container_id=escape(self.default_parameters['container_id']), + instance_id=escape(self.default_parameters['instance_id']), + health_status=escape(self.provisioning_not_ready_status), + health_detail_subsection=health_detail_subsection) + def test_send_ready_signal_sends_post_request(self): with mock.patch.object( azure_helper.GoalStateHealthReporter, @@ -514,55 +554,130 @@ class TestGoalStateHealthReporter(CiTestCase): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) reporter = azure_helper.GoalStateHealthReporter( azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), - client, self.test_endpoint) + client, self.test_azure_endpoint) reporter.send_ready_signal() self.assertEqual(1, self.post.call_count) self.assertEqual( mock.call( - self.test_url, + self.test_health_report_url, + data=m_build_report.return_value, + extra_headers=self.test_default_headers), + self.post.call_args) + + def test_send_failure_signal_sends_post_request(self): + with mock.patch.object( + azure_helper.GoalStateHealthReporter, + 'build_report') as m_build_report: + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + client, self.test_azure_endpoint) + reporter.send_failure_signal( + description=self.provisioning_failure_err_description) + + self.assertEqual(1, self.post.call_count) + self.assertEqual( + mock.call( + self.test_health_report_url, data=m_build_report.return_value, extra_headers=self.test_default_headers), self.post.call_args) - def test_build_report_for_health_document(self): + def test_build_report_for_ready_signal_health_document(self): health_document = self._get_report_ready_health_document() reporter = azure_helper.GoalStateHealthReporter( azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), azure_helper.AzureEndpointHttpClient(mock.MagicMock()), - self.test_endpoint) + self.test_azure_endpoint) generated_health_document = reporter.build_report( incarnation=self.default_parameters['incarnation'], container_id=self.default_parameters['container_id'], instance_id=self.default_parameters['instance_id'], status=self.provisioning_success_status) + self.assertEqual(health_document, generated_health_document) - self.assertIn( - '{}'.format( - str(self.default_parameters['incarnation'])), - generated_health_document) - self.assertIn( - ''.join([ - '', - self.default_parameters['container_id'], - '']), - generated_health_document) - self.assertIn( - ''.join([ - '', - self.default_parameters['instance_id'], - '']), - generated_health_document) - self.assertIn( - ''.join([ - '', - self.provisioning_success_status, - '']), - generated_health_document + + generated_xroot = ElementTree.fromstring(generated_health_document) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, './GoalStateIncarnation'), + str(self.default_parameters['incarnation'])) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, './Container/ContainerId'), + str(self.default_parameters['container_id'])) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/InstanceId'), + str(self.default_parameters['instance_id'])) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/State'), + escape(self.provisioning_success_status)) + self.assertIsNone( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details')) + self.assertIsNone( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/SubStatus')) + self.assertIsNone( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/Description') ) - self.assertNotIn('
', generated_health_document) - self.assertNotIn('', generated_health_document) - self.assertNotIn('', generated_health_document) + + def test_build_report_for_failure_signal_health_document(self): + health_document = self._get_report_failure_health_document() + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + azure_helper.AzureEndpointHttpClient(mock.MagicMock()), + self.test_azure_endpoint) + generated_health_document = reporter.build_report( + incarnation=self.default_parameters['incarnation'], + container_id=self.default_parameters['container_id'], + instance_id=self.default_parameters['instance_id'], + status=self.provisioning_not_ready_status, + substatus=self.provisioning_failure_substatus, + description=self.provisioning_failure_err_description) + + self.assertEqual(health_document, generated_health_document) + + generated_xroot = ElementTree.fromstring(generated_health_document) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, './GoalStateIncarnation'), + str(self.default_parameters['incarnation'])) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, './Container/ContainerId'), + self.default_parameters['container_id']) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/InstanceId'), + self.default_parameters['instance_id']) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/State'), + escape(self.provisioning_not_ready_status)) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/' + 'SubStatus'), + escape(self.provisioning_failure_substatus)) + self.assertEqual( + self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/' + 'Description'), + escape(self.provisioning_failure_err_description)) def test_send_ready_signal_calls_build_report(self): with mock.patch.object( @@ -571,7 +686,7 @@ class TestGoalStateHealthReporter(CiTestCase): reporter = azure_helper.GoalStateHealthReporter( azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), azure_helper.AzureEndpointHttpClient(mock.MagicMock()), - self.test_endpoint) + self.test_azure_endpoint) reporter.send_ready_signal() self.assertEqual(1, m_build_report.call_count) @@ -583,6 +698,131 @@ class TestGoalStateHealthReporter(CiTestCase): status=self.provisioning_success_status), m_build_report.call_args) + def test_send_failure_signal_calls_build_report(self): + with mock.patch.object( + azure_helper.GoalStateHealthReporter, 'build_report' + ) as m_build_report: + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + azure_helper.AzureEndpointHttpClient(mock.MagicMock()), + self.test_azure_endpoint) + reporter.send_failure_signal( + description=self.provisioning_failure_err_description) + + self.assertEqual(1, m_build_report.call_count) + self.assertEqual( + mock.call( + incarnation=self.default_parameters['incarnation'], + container_id=self.default_parameters['container_id'], + instance_id=self.default_parameters['instance_id'], + status=self.provisioning_not_ready_status, + substatus=self.provisioning_failure_substatus, + description=self.provisioning_failure_err_description), + m_build_report.call_args) + + def test_build_report_escapes_chars(self): + incarnation = 'jd8\'9*&^<\'A>' + instance_id = 'Opo>>>jas\'&d;[p&fp\"a<&aa\'sd!@&!)((*<&>' + health_substatus = '&as\"d<d<\'^@!5&6<7' + health_description = '&&&>!#$\"&&><>&\"sd<67<]>>' + + health_detail_subsection = \ + self._get_formatted_health_detail_subsection_xml_string( + health_substatus=escape(health_substatus), + health_description=escape(health_description)) + health_document = self._get_formatted_health_report_xml_string( + incarnation=escape(incarnation), + container_id=escape(container_id), + instance_id=escape(instance_id), + health_status=escape(health_status), + health_detail_subsection=health_detail_subsection) + + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + azure_helper.AzureEndpointHttpClient(mock.MagicMock()), + self.test_azure_endpoint) + generated_health_document = reporter.build_report( + incarnation=incarnation, + container_id=container_id, + instance_id=instance_id, + status=health_status, + substatus=health_substatus, + description=health_description) + + self.assertEqual(health_document, generated_health_document) + + def test_build_report_conforms_to_length_limits(self): + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + azure_helper.AzureEndpointHttpClient(mock.MagicMock()), + self.test_azure_endpoint) + long_err_msg = 'a9&ea8>>>e as1< d\"q2*&(^%\'a=5<' * 100 + generated_health_document = reporter.build_report( + incarnation=self.default_parameters['incarnation'], + container_id=self.default_parameters['container_id'], + instance_id=self.default_parameters['instance_id'], + status=self.provisioning_not_ready_status, + substatus=self.provisioning_failure_substatus, + description=long_err_msg) + + generated_xroot = ElementTree.fromstring(generated_health_document) + generated_health_report_description = self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/Description') + self.assertEqual( + len(unescape(generated_health_report_description)), + HEALTH_REPORT_DESCRIPTION_TRIM_LEN) + + def test_trim_description_then_escape_conforms_to_len_limits_worst_case( + self): + """When unescaped characters are XML-escaped, the length increases. + Char Escape String + < < + > > + " " + ' ' + & & + + We (step 1) trim the health report XML's description field, + and then (step 2) XML-escape the health report XML's description field. + + The health report XML's description field limit within cloud-init + is HEALTH_REPORT_DESCRIPTION_TRIM_LEN. + + The Azure platform's limit on the health report XML's description field + is 4096 chars. + + For worst-case chars, there is a 5x blowup in length + when the chars are XML-escaped. + ' and " when XML-escaped have a 5x blowup. + + Ensure that (1) trimming and then (2) XML-escaping does not blow past + the Azure platform's limit for health report XML's description field + (4096 chars). + """ + reporter = azure_helper.GoalStateHealthReporter( + azure_helper.GoalState(mock.MagicMock(), mock.MagicMock()), + azure_helper.AzureEndpointHttpClient(mock.MagicMock()), + self.test_azure_endpoint) + long_err_msg = '\'\"' * 10000 + generated_health_document = reporter.build_report( + incarnation=self.default_parameters['incarnation'], + container_id=self.default_parameters['container_id'], + instance_id=self.default_parameters['instance_id'], + status=self.provisioning_not_ready_status, + substatus=self.provisioning_failure_substatus, + description=long_err_msg) + + generated_xroot = ElementTree.fromstring(generated_health_document) + generated_health_report_description = self._text_from_xpath_in_xroot( + generated_xroot, + './Container/RoleInstanceList/Role/Health/Details/Description') + # The escaped description string should be less than + # the Azure platform limit for the escaped description string. + self.assertLessEqual(len(generated_health_report_description), 4096) + class TestWALinuxAgentShim(CiTestCase): @@ -598,7 +838,7 @@ class TestWALinuxAgentShim(CiTestCase): self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) self.OpenSSLManager = patches.enter_context( - mock.patch.object(azure_helper, 'OpenSSLManager')) + mock.patch.object(azure_helper, 'OpenSSLManager', autospec=True)) patches.enter_context( mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) @@ -609,24 +849,47 @@ class TestWALinuxAgentShim(CiTestCase): self.GoalState.return_value.container_id = self.test_container_id self.GoalState.return_value.instance_id = self.test_instance_id - def test_http_client_does_not_use_certificate(self): + def test_http_client_does_not_use_certificate_for_report_ready(self): shim = wa_shim() shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(None)], self.AzureEndpointHttpClient.call_args_list) + def test_http_client_does_not_use_certificate_for_report_failure(self): + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + self.assertEqual( + [mock.call(None)], + self.AzureEndpointHttpClient.call_args_list) + def test_correct_url_used_for_goalstate_during_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' shim = wa_shim() shim.register_with_azure_and_fetch_data() - get = self.AzureEndpointHttpClient.return_value.get + m_get = self.AzureEndpointHttpClient.return_value.get + self.assertEqual( + [mock.call('http://test_endpoint/machine/?comp=goalstate')], + m_get.call_args_list) + self.assertEqual( + [mock.call( + m_get.return_value.contents, + self.AzureEndpointHttpClient.return_value, + False + )], + self.GoalState.call_args_list) + + def test_correct_url_used_for_goalstate_during_report_failure(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + m_get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( [mock.call('http://test_endpoint/machine/?comp=goalstate')], - get.call_args_list) + m_get.call_args_list) self.assertEqual( [mock.call( - get.return_value.contents, + m_get.return_value.contents, self.AzureEndpointHttpClient.return_value, False )], @@ -670,6 +933,16 @@ class TestWALinuxAgentShim(CiTestCase): self.AzureEndpointHttpClient.return_value.post .call_args_list) + def test_correct_url_used_for_report_failure(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + expected_url = 'http://test_endpoint/machine?comp=health' + self.assertEqual( + [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], + self.AzureEndpointHttpClient.return_value.post + .call_args_list) + def test_goal_state_values_used_for_report_ready(self): shim = wa_shim() shim.register_with_azure_and_fetch_data() @@ -681,44 +954,128 @@ class TestWALinuxAgentShim(CiTestCase): self.assertIn(self.test_container_id, posted_document) self.assertIn(self.test_instance_id, posted_document) - def test_xml_elems_in_report_ready(self): + def test_goal_state_values_used_for_report_failure(self): + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + posted_document = ( + self.AzureEndpointHttpClient.return_value.post + .call_args[1]['data'] + ) + self.assertIn(self.test_incarnation, posted_document) + self.assertIn(self.test_container_id, posted_document) + self.assertIn(self.test_instance_id, posted_document) + + def test_xml_elems_in_report_ready_post(self): shim = wa_shim() shim.register_with_azure_and_fetch_data() health_document = HEALTH_REPORT_XML_TEMPLATE.format( - incarnation=self.test_incarnation, - container_id=self.test_container_id, - instance_id=self.test_instance_id, - health_status='Ready', + incarnation=escape(self.test_incarnation), + container_id=escape(self.test_container_id), + instance_id=escape(self.test_instance_id), + health_status=escape('Ready'), health_detail_subsection='') posted_document = ( self.AzureEndpointHttpClient.return_value.post .call_args[1]['data']) self.assertEqual(health_document, posted_document) + def test_xml_elems_in_report_failure_post(self): + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + health_document = HEALTH_REPORT_XML_TEMPLATE.format( + incarnation=escape(self.test_incarnation), + container_id=escape(self.test_container_id), + instance_id=escape(self.test_instance_id), + health_status=escape('NotReady'), + health_detail_subsection=HEALTH_DETAIL_SUBSECTION_XML_TEMPLATE + .format( + health_substatus=escape('ProvisioningFailed'), + health_description=escape('TestDesc'))) + posted_document = ( + self.AzureEndpointHttpClient.return_value.post + .call_args[1]['data']) + self.assertEqual(health_document, posted_document) + + @mock.patch.object(azure_helper, 'GoalStateHealthReporter', autospec=True) + def test_register_with_azure_and_fetch_data_calls_send_ready_signal( + self, m_goal_state_health_reporter): + shim = wa_shim() + shim.register_with_azure_and_fetch_data() + self.assertEqual( + 1, + m_goal_state_health_reporter.return_value.send_ready_signal + .call_count) + + @mock.patch.object(azure_helper, 'GoalStateHealthReporter', autospec=True) + def test_register_with_azure_and_report_failure_calls_send_failure_signal( + self, m_goal_state_health_reporter): + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + m_goal_state_health_reporter.return_value.send_failure_signal \ + .assert_called_once_with(description='TestDesc') + + def test_register_with_azure_and_report_failure_does_not_need_certificates( + self): + shim = wa_shim() + with mock.patch.object( + shim, '_fetch_goal_state_from_azure', autospec=True + ) as m_fetch_goal_state_from_azure: + shim.register_with_azure_and_report_failure(description='TestDesc') + m_fetch_goal_state_from_azure.assert_called_once_with( + need_certificate=False) + def test_clean_up_can_be_called_at_any_time(self): shim = wa_shim() shim.clean_up() + def test_openssl_manager_not_instantiated_by_shim_report_status(self): + shim = wa_shim() + shim.register_with_azure_and_fetch_data() + shim.register_with_azure_and_report_failure(description='TestDesc') + shim.clean_up() + self.OpenSSLManager.assert_not_called() + def test_clean_up_after_report_ready(self): shim = wa_shim() shim.register_with_azure_and_fetch_data() shim.clean_up() - self.assertEqual( - 0, self.OpenSSLManager.return_value.clean_up.call_count) + self.OpenSSLManager.return_value.clean_up.assert_not_called() + + def test_clean_up_after_report_failure(self): + shim = wa_shim() + shim.register_with_azure_and_report_failure(description='TestDesc') + shim.clean_up() + self.OpenSSLManager.return_value.clean_up.assert_not_called() def test_fetch_goalstate_during_report_ready_raises_exc_on_get_exc(self): self.AzureEndpointHttpClient.return_value.get \ - .side_effect = (SentinelException) + .side_effect = SentinelException shim = wa_shim() self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) + def test_fetch_goalstate_during_report_failure_raises_exc_on_get_exc(self): + self.AzureEndpointHttpClient.return_value.get \ + .side_effect = SentinelException + shim = wa_shim() + self.assertRaises(SentinelException, + shim.register_with_azure_and_report_failure, + description='TestDesc') + def test_fetch_goalstate_during_report_ready_raises_exc_on_parse_exc(self): self.GoalState.side_effect = SentinelException shim = wa_shim() self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) + def test_fetch_goalstate_during_report_failure_raises_exc_on_parse_exc( + self): + self.GoalState.side_effect = SentinelException + shim = wa_shim() + self.assertRaises(SentinelException, + shim.register_with_azure_and_report_failure, + description='TestDesc') + def test_failure_to_send_report_ready_health_doc_bubbles_up(self): self.AzureEndpointHttpClient.return_value.post \ .side_effect = SentinelException @@ -726,56 +1083,132 @@ class TestWALinuxAgentShim(CiTestCase): self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) + def test_failure_to_send_report_failure_health_doc_bubbles_up(self): + self.AzureEndpointHttpClient.return_value.post \ + .side_effect = SentinelException + shim = wa_shim() + self.assertRaises(SentinelException, + shim.register_with_azure_and_report_failure, + description='TestDesc') + class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase): - @mock.patch.object(azure_helper, 'WALinuxAgentShim') - def test_data_from_shim_returned(self, shim): + def setUp(self): + super(TestGetMetadataGoalStateXMLAndReportReadyToFabric, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.m_shim = patches.enter_context( + mock.patch.object(azure_helper, 'WALinuxAgentShim')) + + def test_data_from_shim_returned(self): ret = azure_helper.get_metadata_from_fabric() self.assertEqual( - shim.return_value.register_with_azure_and_fetch_data.return_value, + self.m_shim.return_value.register_with_azure_and_fetch_data + .return_value, ret) - @mock.patch.object(azure_helper, 'WALinuxAgentShim') - def test_success_calls_clean_up(self, shim): + def test_success_calls_clean_up(self): azure_helper.get_metadata_from_fabric() - self.assertEqual(1, shim.return_value.clean_up.call_count) + self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) - @mock.patch.object(azure_helper, 'WALinuxAgentShim') def test_failure_in_registration_propagates_exc_and_calls_clean_up( - self, shim): - shim.return_value.register_with_azure_and_fetch_data.side_effect = ( - SentinelException) + self): + self.m_shim.return_value.register_with_azure_and_fetch_data \ + .side_effect = SentinelException self.assertRaises(SentinelException, azure_helper.get_metadata_from_fabric) - self.assertEqual(1, shim.return_value.clean_up.call_count) + self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) - @mock.patch.object(azure_helper, 'WALinuxAgentShim') - def test_calls_shim_register_with_azure_and_fetch_data(self, shim): + def test_calls_shim_register_with_azure_and_fetch_data(self): m_pubkey_info = mock.MagicMock() azure_helper.get_metadata_from_fabric(pubkey_info=m_pubkey_info) self.assertEqual( 1, - shim.return_value + self.m_shim.return_value .register_with_azure_and_fetch_data.call_count) self.assertEqual( mock.call(pubkey_info=m_pubkey_info), - shim.return_value + self.m_shim.return_value .register_with_azure_and_fetch_data.call_args) - @mock.patch.object(azure_helper, 'WALinuxAgentShim') - def test_instantiates_shim_with_kwargs(self, shim): + def test_instantiates_shim_with_kwargs(self): m_fallback_lease_file = mock.MagicMock() m_dhcp_options = mock.MagicMock() azure_helper.get_metadata_from_fabric( fallback_lease_file=m_fallback_lease_file, dhcp_opts=m_dhcp_options) - self.assertEqual(1, shim.call_count) + self.assertEqual(1, self.m_shim.call_count) self.assertEqual( mock.call( fallback_lease_file=m_fallback_lease_file, dhcp_options=m_dhcp_options), - shim.call_args) + self.m_shim.call_args) + + +class TestGetMetadataGoalStateXMLAndReportFailureToFabric(CiTestCase): + + def setUp(self): + super( + TestGetMetadataGoalStateXMLAndReportFailureToFabric, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.m_shim = patches.enter_context( + mock.patch.object(azure_helper, 'WALinuxAgentShim')) + + def test_success_calls_clean_up(self): + azure_helper.report_failure_to_fabric() + self.assertEqual( + 1, + self.m_shim.return_value.clean_up.call_count) + + def test_failure_in_shim_report_failure_propagates_exc_and_calls_clean_up( + self): + self.m_shim.return_value.register_with_azure_and_report_failure \ + .side_effect = SentinelException + self.assertRaises(SentinelException, + azure_helper.report_failure_to_fabric) + self.assertEqual( + 1, + self.m_shim.return_value.clean_up.call_count) + + def test_report_failure_to_fabric_with_desc_calls_shim_report_failure( + self): + azure_helper.report_failure_to_fabric(description='TestDesc') + self.m_shim.return_value.register_with_azure_and_report_failure \ + .assert_called_once_with(description='TestDesc') + + def test_report_failure_to_fabric_with_no_desc_calls_shim_report_failure( + self): + azure_helper.report_failure_to_fabric() + # default err message description should be shown to the user + # if no description is passed in + self.m_shim.return_value.register_with_azure_and_report_failure \ + .assert_called_once_with( + description=azure_helper + .DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE) + + def test_report_failure_to_fabric_empty_desc_calls_shim_report_failure( + self): + azure_helper.report_failure_to_fabric(description='') + # default err message description should be shown to the user + # if an empty description is passed in + self.m_shim.return_value.register_with_azure_and_report_failure \ + .assert_called_once_with( + description=azure_helper + .DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE) + + def test_instantiates_shim_with_kwargs(self): + m_fallback_lease_file = mock.MagicMock() + m_dhcp_options = mock.MagicMock() + azure_helper.report_failure_to_fabric( + fallback_lease_file=m_fallback_lease_file, + dhcp_opts=m_dhcp_options) + self.m_shim.assert_called_once_with( + fallback_lease_file=m_fallback_lease_file, + dhcp_options=m_dhcp_options) class TestExtractIpAddressFromNetworkd(CiTestCase): -- cgit v1.2.3 From 6df0230b1201d6bed8661b19d8f3758797635377 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 18 Nov 2020 10:02:56 -0800 Subject: Azure helper: Increase Azure Endpoint HTTP retries (#619) Increase Azure Endpoint HTTP retries to handle occasional platform network blips. Introduce a common method http_with_retries in the azure.py helper, which will serve as the common HTTP request handler for all HTTP requests with the Azure endpoint. This method has builtin retries and reporting diagnostics logic. --- cloudinit/sources/helpers/azure.py | 55 ++++- .../unittests/test_datasource/test_azure_helper.py | 227 +++++++++++++++++---- 2 files changed, 241 insertions(+), 41 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 951c7a10..2b3303c7 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -284,6 +284,54 @@ def _get_dhcp_endpoint_option_name(): return azure_endpoint +@azure_ds_telemetry_reporter +def http_with_retries(url, **kwargs) -> str: + """Wrapper around url_helper.readurl() with custom telemetry logging + that url_helper.readurl() does not provide. + """ + exc = None + + max_readurl_attempts = 240 + default_readurl_timeout = 5 + periodic_logging_attempts = 12 + + if 'timeout' not in kwargs: + kwargs['timeout'] = default_readurl_timeout + + # remove kwargs that cause url_helper.readurl to retry, + # since we are already implementing our own retry logic. + if kwargs.pop('retries', None): + LOG.warning( + 'Ignoring retries kwarg passed in for ' + 'communication with Azure endpoint.') + if kwargs.pop('infinite', None): + LOG.warning( + 'Ignoring infinite kwarg passed in for communication ' + 'with Azure endpoint.') + + for attempt in range(1, max_readurl_attempts + 1): + try: + ret = url_helper.readurl(url, **kwargs) + + report_diagnostic_event( + 'Successful HTTP request with Azure endpoint %s after ' + '%d attempts' % (url, attempt), + logger_func=LOG.debug) + + return ret + + except Exception as e: + exc = e + if attempt % periodic_logging_attempts == 0: + report_diagnostic_event( + 'Failed HTTP request with Azure endpoint %s during ' + 'attempt %d with exception: %s' % + (url, attempt, e), + logger_func=LOG.debug) + + raise exc + + class AzureEndpointHttpClient: headers = { @@ -302,16 +350,15 @@ class AzureEndpointHttpClient: if secure: headers = self.headers.copy() headers.update(self.extra_secure_headers) - return url_helper.readurl(url, headers=headers, - timeout=5, retries=10, sec_between=5) + return http_with_retries(url, headers=headers) def post(self, url, data=None, extra_headers=None): headers = self.headers if extra_headers is not None: headers = self.headers.copy() headers.update(extra_headers) - return url_helper.readurl(url, data=data, headers=headers, - timeout=5, retries=10, sec_between=5) + return http_with_retries( + url, data=data, headers=headers) class InvalidGoalStateXMLException(Exception): diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index adf68857..b8899807 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +import copy import os import re import unittest @@ -291,29 +292,25 @@ class TestAzureEndpointHttpClient(CiTestCase): super(TestAzureEndpointHttpClient, self).setUp() patches = ExitStack() self.addCleanup(patches.close) - - self.readurl = patches.enter_context( - mock.patch.object(azure_helper.url_helper, 'readurl')) - patches.enter_context( - mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) + self.m_http_with_retries = patches.enter_context( + mock.patch.object(azure_helper, 'http_with_retries')) def test_non_secure_get(self): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) url = 'MyTestUrl' response = client.get(url, secure=False) - self.assertEqual(1, self.readurl.call_count) - self.assertEqual(self.readurl.return_value, response) + self.assertEqual(1, self.m_http_with_retries.call_count) + self.assertEqual(self.m_http_with_retries.return_value, response) self.assertEqual( - mock.call(url, headers=self.regular_headers, - timeout=5, retries=10, sec_between=5), - self.readurl.call_args) + mock.call(url, headers=self.regular_headers), + self.m_http_with_retries.call_args) def test_non_secure_get_raises_exception(self): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.readurl.side_effect = SentinelException url = 'MyTestUrl' - with self.assertRaises(SentinelException): - client.get(url, secure=False) + self.m_http_with_retries.side_effect = SentinelException + self.assertRaises(SentinelException, client.get, url, secure=False) + self.assertEqual(1, self.m_http_with_retries.call_count) def test_secure_get(self): url = 'MyTestUrl' @@ -325,39 +322,37 @@ class TestAzureEndpointHttpClient(CiTestCase): }) client = azure_helper.AzureEndpointHttpClient(m_certificate) response = client.get(url, secure=True) - self.assertEqual(1, self.readurl.call_count) - self.assertEqual(self.readurl.return_value, response) + self.assertEqual(1, self.m_http_with_retries.call_count) + self.assertEqual(self.m_http_with_retries.return_value, response) self.assertEqual( - mock.call(url, headers=expected_headers, - timeout=5, retries=10, sec_between=5), - self.readurl.call_args) + mock.call(url, headers=expected_headers), + self.m_http_with_retries.call_args) def test_secure_get_raises_exception(self): url = 'MyTestUrl' client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.readurl.side_effect = SentinelException - with self.assertRaises(SentinelException): - client.get(url, secure=True) + self.m_http_with_retries.side_effect = SentinelException + self.assertRaises(SentinelException, client.get, url, secure=True) + self.assertEqual(1, self.m_http_with_retries.call_count) def test_post(self): m_data = mock.MagicMock() url = 'MyTestUrl' client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) response = client.post(url, data=m_data) - self.assertEqual(1, self.readurl.call_count) - self.assertEqual(self.readurl.return_value, response) + self.assertEqual(1, self.m_http_with_retries.call_count) + self.assertEqual(self.m_http_with_retries.return_value, response) self.assertEqual( - mock.call(url, data=m_data, headers=self.regular_headers, - timeout=5, retries=10, sec_between=5), - self.readurl.call_args) + mock.call(url, data=m_data, headers=self.regular_headers), + self.m_http_with_retries.call_args) def test_post_raises_exception(self): m_data = mock.MagicMock() url = 'MyTestUrl' client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.readurl.side_effect = SentinelException - with self.assertRaises(SentinelException): - client.post(url, data=m_data) + self.m_http_with_retries.side_effect = SentinelException + self.assertRaises(SentinelException, client.post, url, data=m_data) + self.assertEqual(1, self.m_http_with_retries.call_count) def test_post_with_extra_headers(self): url = 'MyTestUrl' @@ -366,21 +361,179 @@ class TestAzureEndpointHttpClient(CiTestCase): client.post(url, extra_headers=extra_headers) expected_headers = self.regular_headers.copy() expected_headers.update(extra_headers) - self.assertEqual(1, self.readurl.call_count) + self.assertEqual(1, self.m_http_with_retries.call_count) self.assertEqual( - mock.call(mock.ANY, data=mock.ANY, headers=expected_headers, - timeout=5, retries=10, sec_between=5), - self.readurl.call_args) + mock.call(url, data=mock.ANY, headers=expected_headers), + self.m_http_with_retries.call_args) def test_post_with_sleep_with_extra_headers_raises_exception(self): m_data = mock.MagicMock() url = 'MyTestUrl' extra_headers = {'test': 'header'} client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.readurl.side_effect = SentinelException - with self.assertRaises(SentinelException): - client.post( - url, data=m_data, extra_headers=extra_headers) + self.m_http_with_retries.side_effect = SentinelException + self.assertRaises( + SentinelException, client.post, + url, data=m_data, extra_headers=extra_headers) + self.assertEqual(1, self.m_http_with_retries.call_count) + + +class TestAzureHelperHttpWithRetries(CiTestCase): + + with_logs = True + + max_readurl_attempts = 240 + default_readurl_timeout = 5 + periodic_logging_attempts = 12 + + def setUp(self): + super(TestAzureHelperHttpWithRetries, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.m_readurl = patches.enter_context( + mock.patch.object( + azure_helper.url_helper, 'readurl', mock.MagicMock())) + patches.enter_context( + mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) + + def test_http_with_retries(self): + self.m_readurl.return_value = 'TestResp' + self.assertEqual( + azure_helper.http_with_retries('testurl'), + self.m_readurl.return_value) + self.assertEqual(self.m_readurl.call_count, 1) + + def test_http_with_retries_propagates_readurl_exc_and_logs_exc( + self): + self.m_readurl.side_effect = SentinelException + + self.assertRaises( + SentinelException, azure_helper.http_with_retries, 'testurl') + self.assertEqual(self.m_readurl.call_count, self.max_readurl_attempts) + + self.assertIsNotNone( + re.search( + r'Failed HTTP request with Azure endpoint \S* during ' + r'attempt \d+ with exception: \S*', + self.logs.getvalue())) + self.assertIsNone( + re.search( + r'Successful HTTP request with Azure endpoint \S* after ' + r'\d+ attempts', + self.logs.getvalue())) + + def test_http_with_retries_delayed_success_due_to_temporary_readurl_exc( + self): + self.m_readurl.side_effect = \ + [SentinelException] * self.periodic_logging_attempts + \ + ['TestResp'] + self.m_readurl.return_value = 'TestResp' + + response = azure_helper.http_with_retries('testurl') + self.assertEqual( + response, + self.m_readurl.return_value) + self.assertEqual( + self.m_readurl.call_count, + self.periodic_logging_attempts + 1) + + def test_http_with_retries_long_delay_logs_periodic_failure_msg(self): + self.m_readurl.side_effect = \ + [SentinelException] * self.periodic_logging_attempts + \ + ['TestResp'] + self.m_readurl.return_value = 'TestResp' + + azure_helper.http_with_retries('testurl') + + self.assertEqual( + self.m_readurl.call_count, + self.periodic_logging_attempts + 1) + self.assertIsNotNone( + re.search( + r'Failed HTTP request with Azure endpoint \S* during ' + r'attempt \d+ with exception: \S*', + self.logs.getvalue())) + self.assertIsNotNone( + re.search( + r'Successful HTTP request with Azure endpoint \S* after ' + r'\d+ attempts', + self.logs.getvalue())) + + def test_http_with_retries_short_delay_does_not_log_periodic_failure_msg( + self): + self.m_readurl.side_effect = \ + [SentinelException] * \ + (self.periodic_logging_attempts - 1) + \ + ['TestResp'] + self.m_readurl.return_value = 'TestResp' + + azure_helper.http_with_retries('testurl') + self.assertEqual( + self.m_readurl.call_count, + self.periodic_logging_attempts) + + self.assertIsNone( + re.search( + r'Failed HTTP request with Azure endpoint \S* during ' + r'attempt \d+ with exception: \S*', + self.logs.getvalue())) + self.assertIsNotNone( + re.search( + r'Successful HTTP request with Azure endpoint \S* after ' + r'\d+ attempts', + self.logs.getvalue())) + + def test_http_with_retries_calls_url_helper_readurl_with_args_kwargs(self): + testurl = mock.MagicMock() + kwargs = { + 'headers': mock.MagicMock(), + 'data': mock.MagicMock(), + # timeout kwarg should not be modified or deleted if present + 'timeout': mock.MagicMock() + } + azure_helper.http_with_retries(testurl, **kwargs) + self.m_readurl.assert_called_once_with(testurl, **kwargs) + + def test_http_with_retries_adds_timeout_kwarg_if_not_present(self): + testurl = mock.MagicMock() + kwargs = { + 'headers': mock.MagicMock(), + 'data': mock.MagicMock() + } + expected_kwargs = copy.deepcopy(kwargs) + expected_kwargs['timeout'] = self.default_readurl_timeout + + azure_helper.http_with_retries(testurl, **kwargs) + self.m_readurl.assert_called_once_with(testurl, **expected_kwargs) + + def test_http_with_retries_deletes_retries_kwargs_passed_in( + self): + """http_with_retries already implements retry logic, + so url_helper.readurl should not have retries. + http_with_retries should delete kwargs that + cause url_helper.readurl to retry. + """ + testurl = mock.MagicMock() + kwargs = { + 'headers': mock.MagicMock(), + 'data': mock.MagicMock(), + 'timeout': mock.MagicMock(), + 'retries': mock.MagicMock(), + 'infinite': mock.MagicMock() + } + expected_kwargs = copy.deepcopy(kwargs) + expected_kwargs.pop('retries', None) + expected_kwargs.pop('infinite', None) + + azure_helper.http_with_retries(testurl, **kwargs) + self.m_readurl.assert_called_once_with(testurl, **expected_kwargs) + self.assertIn( + 'retries kwarg passed in for communication with Azure endpoint.', + self.logs.getvalue()) + self.assertIn( + 'infinite kwarg passed in for communication with Azure endpoint.', + self.logs.getvalue()) class TestOpenSSLManager(CiTestCase): -- cgit v1.2.3 From 49d5de92d60af4ef483a67bd4c75148e6fdc58a3 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Nov 2020 00:09:49 -0500 Subject: tox.ini: only select "ci" marked tests for CI runs (#677) `-k` will select tests or marks whose names match. `-m` selects only tests which have the specified mark, so is more appropriate. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4320ab87..c86d38e9 100644 --- a/tox.ini +++ b/tox.ini @@ -151,7 +151,7 @@ commands = {[testenv:integration-tests]commands} passenv = {[testenv:integration-tests]passenv} deps = {[testenv:integration-tests]deps} setenv = - PYTEST_ADDOPTS="-k ci" + PYTEST_ADDOPTS="-m ci" [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped -- cgit v1.2.3 From eea754492f074e00b601cf77aa278e3623857c5a Mon Sep 17 00:00:00 2001 From: Anh Vo Date: Thu, 19 Nov 2020 00:35:46 -0500 Subject: DataSourceAzure: update password for defuser if exists (#671) cc_set_password will only update the password for the default user if cfg['password'] is set. The existing code of datasource Azure will fail to update the default user's password because it does not set that metadata. If the default user doesn't exist in the image, the current code works fine because the password is set during user create and not in cc_set_password --- cloudinit/sources/DataSourceAzure.py | 2 +- tests/unittests/test_datasource/test_azure.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ab139b8d..f777a007 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1391,7 +1391,7 @@ def read_azure_ovf(contents): if password: defuser['lock_passwd'] = False if DEF_PASSWD_REDACTION != password: - defuser['passwd'] = encrypt_pass(password) + defuser['passwd'] = cfg['password'] = encrypt_pass(password) if defuser: cfg['system_info'] = {'default_user': defuser} diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index d9752ab7..534314aa 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1080,6 +1080,9 @@ scbus-1 on xpt0 bus 0 crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) + # the same hashed value should also be present in cfg['password'] + self.assertEqual(defuser['passwd'], dsrc.cfg['password']) + def test_user_not_locked_if_password_redacted(self): odata = {'HostName': "myhost", 'UserName': "myuser", 'UserPassword': dsaz.DEF_PASSWD_REDACTION} -- cgit v1.2.3 From bfaee8cc9b8fd23dbc118ae548fc2ca695a0d707 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Nov 2020 12:42:49 -0500 Subject: integration_tests: restore emission of settings to log (#657) --- tests/integration_tests/clouds.py | 11 +++++++++++ tests/integration_tests/conftest.py | 1 + tests/integration_tests/instances.py | 11 ----------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 71d4e85d..fe89c0c6 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -31,6 +31,17 @@ class IntegrationCloud(ABC): self.cloud_instance = self._get_cloud_instance() self.image_id = self._get_initial_image() + def emit_settings_to_log(self) -> None: + log.info( + "\n".join( + ["Settings:"] + + [ + "{}={}".format(key, getattr(self.settings, key)) + for key in sorted(self.settings.current_settings) + ] + ) + ) + @abstractmethod def _get_cloud_instance(self): raise NotImplementedError diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 34e674e9..eacb2ae2 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -62,6 +62,7 @@ def session_cloud(): ) cloud = platforms[integration_settings.PLATFORM]() + cloud.emit_settings_to_log() yield cloud cloud.destroy() diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index d64c1ab2..0db7c07b 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -29,17 +29,6 @@ class IntegrationInstance: self.instance = instance self.settings = settings - def emit_settings_to_log(self) -> None: - log.info( - "\n".join( - ["Settings:"] - + [ - "{}={}".format(key, getattr(self.settings, key)) - for key in sorted(self.settings.current_settings) - ] - ) - ) - def destroy(self): self.instance.delete() -- cgit v1.2.3 From 9707a08a82161cd4129f6cdd10978cde50bea747 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 19 Nov 2020 11:54:35 -0600 Subject: Make mount in place for tests work (#667) IMAGE_SOURCE = 'IN_PLACE' wasn't working previously. Replaced LXD launch with an init, then mount, then start. --- tests/integration_tests/clouds.py | 44 ++++++++++++++++++++++++++++++++++-- tests/integration_tests/conftest.py | 2 +- tests/integration_tests/instances.py | 17 -------------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index fe89c0c6..2841261b 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -4,6 +4,8 @@ import logging from pycloudlib import EC2, GCE, Azure, OCI, LXD +import cloudinit +from cloudinit.subp import subp from tests.integration_tests import integration_settings from tests.integration_tests.instances import ( IntegrationEc2Instance, @@ -55,6 +57,11 @@ class IntegrationCloud(ABC): pass return image_id + def _perform_launch(self, launch_kwargs): + pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) + pycloudlib_instance.wait(raise_on_cloudinit_failure=False) + return pycloudlib_instance + def launch(self, user_data=None, launch_kwargs=None, settings=integration_settings): if self.settings.EXISTING_INSTANCE_ID: @@ -77,8 +84,9 @@ class IntegrationCloud(ABC): "\n".join("{}={}".format(*item) for item in kwargs.items()) ) ) - pycloudlib_instance = self.cloud_instance.launch(**kwargs) - pycloudlib_instance.wait(raise_on_cloudinit_failure=False) + + pycloudlib_instance = self._perform_launch(kwargs) + log.info('Launched instance: %s', pycloudlib_instance) return self.get_instance(pycloudlib_instance, settings) @@ -141,3 +149,35 @@ class LxdContainerCloud(IntegrationCloud): def _get_cloud_instance(self): return LXD(tag='lxd-integration-test') + + def _perform_launch(self, launch_kwargs): + launch_kwargs['inst_type'] = launch_kwargs.pop('instance_type', None) + launch_kwargs.pop('wait') + + pycloudlib_instance = self.cloud_instance.init( + launch_kwargs.pop('name', None), + launch_kwargs.pop('image_id'), + **launch_kwargs + ) + if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + self._mount_source(pycloudlib_instance) + pycloudlib_instance.start(wait=False) + pycloudlib_instance.wait(raise_on_cloudinit_failure=False) + return pycloudlib_instance + + def _mount_source(self, instance): + container_path = '/usr/lib/python3/dist-packages/cloudinit' + format_variables = { + 'name': instance.name, + 'cloudinit_path': cloudinit.__path__[0], + 'container_path': container_path, + } + log.info( + 'Mounting source {cloudinit_path} directly onto LXD container/vm ' + 'named {name} at {container_path}'.format(**format_variables)) + command = ( + 'lxc config device add {name} host-cloud-init disk ' + 'source={cloudinit_path} ' + 'path={container_path}' + ).format(**format_variables) + subp(command.split()) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index eacb2ae2..e31a9192 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -81,7 +81,7 @@ def setup_image(session_cloud): if session_cloud.datasource != 'lxd_container': raise ValueError( 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') - # The mount needs to happen after the instance is launched, so + # The mount needs to happen after the instance is created, so # no further action needed here elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED': client = session_cloud.launch() diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 0db7c07b..67a6fb92 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -5,8 +5,6 @@ from tempfile import NamedTemporaryFile from pycloudlib.instance import BaseInstance -import cloudinit -from cloudinit.subp import subp from tests.integration_tests import integration_settings try: @@ -129,18 +127,3 @@ class IntegrationOciInstance(IntegrationInstance): class IntegrationLxdContainerInstance(IntegrationInstance): use_sudo = False - - def __init__(self, cloud: 'IntegrationCloud', instance: BaseInstance, - settings=integration_settings): - super().__init__(cloud, instance, settings) - if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': - self._mount_source() - - def _mount_source(self): - command = ( - 'lxc config device add {name} host-cloud-init disk ' - 'source={cloudinit_path} ' - 'path=/usr/lib/python3/dist-packages/cloudinit' - ).format( - name=self.instance.name, cloudinit_path=cloudinit.__path__[0]) - subp(command.split()) -- cgit v1.2.3 From 73e704e3690611625e3cda060a7a6a81492af9d2 Mon Sep 17 00:00:00 2001 From: Anh Vo Date: Thu, 19 Nov 2020 13:38:27 -0500 Subject: DataSourceAzure: push dmesg log to KVP (#670) Pushing dmesg log to KVP to help troubleshoot VM boot issues --- cloudinit/sources/helpers/azure.py | 12 +++++++++++- tests/unittests/test_reporting_hyperv.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 2b3303c7..d3055d08 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -224,7 +224,8 @@ def push_log_to_kvp(file_name=CFG_BUILTIN['def_log_file']): based on the file size. The first time this function is called after VM boot, It will push the last n bytes of the log file such that n < MAX_LOG_TO_KVP_LENGTH - If called again on the same boot, it continues from where it left off.""" + If called again on the same boot, it continues from where it left off. + In addition to cloud-init.log, dmesg log will also be collected.""" start_index = get_last_log_byte_pushed_to_kvp_index() @@ -245,6 +246,15 @@ def push_log_to_kvp(file_name=CFG_BUILTIN['def_log_file']): "Exception when dumping log file: %s" % repr(ex), logger_func=LOG.warning) + LOG.debug("Dumping dmesg log to KVP") + try: + out, _ = subp.subp(['dmesg'], decode=False, capture=True) + report_compressed_event("dmesg", out) + except Exception as ex: + report_diagnostic_event( + "Exception when dumping dmesg log: %s" % repr(ex), + logger_func=LOG.warning) + @azure_ds_telemetry_reporter def get_last_log_byte_pushed_to_kvp_index(): diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index 8f7b3694..9324b78d 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -230,8 +230,33 @@ class TextKvpReporter(CiTestCase): instantiated_handler_registry.unregister_item("telemetry", force=False) + @mock.patch('cloudinit.sources.helpers.azure.report_compressed_event') + @mock.patch('cloudinit.sources.helpers.azure.report_diagnostic_event') + @mock.patch('cloudinit.subp.subp') + def test_push_log_to_kvp_exception_handling(self, m_subp, m_diag, m_com): + reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) + try: + instantiated_handler_registry.register_item("telemetry", reporter) + log_file = self.tmp_path("cloud-init.log") + azure.MAX_LOG_TO_KVP_LENGTH = 100 + azure.LOG_PUSHED_TO_KVP_INDEX_FILE = self.tmp_path( + 'log_pushed_to_kvp') + with open(log_file, "w") as f: + log_content = "A" * 50 + "B" * 100 + f.write(log_content) + + m_com.side_effect = Exception("Mock Exception") + azure.push_log_to_kvp(log_file) + + # exceptions will trigger diagnostic reporting calls + self.assertEqual(m_diag.call_count, 3) + finally: + instantiated_handler_registry.unregister_item("telemetry", + force=False) + + @mock.patch('cloudinit.subp.subp') @mock.patch.object(LogHandler, 'publish_event') - def test_push_log_to_kvp(self, publish_event): + def test_push_log_to_kvp(self, publish_event, m_subp): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) try: instantiated_handler_registry.register_item("telemetry", reporter) @@ -249,6 +274,10 @@ class TextKvpReporter(CiTestCase): f.write(extra_content) azure.push_log_to_kvp(log_file) + # make sure dmesg is called every time + m_subp.assert_called_with( + ['dmesg'], capture=True, decode=False) + for call_arg in publish_event.call_args_list: event = call_arg[0][0] self.assertNotEqual( -- cgit v1.2.3 From 12ef7541c2d0c6b2cd510f95fda53ca9c8333064 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Thu, 19 Nov 2020 23:19:16 +0100 Subject: cc_resizefs on FreeBSD: Fix _can_skip_ufs_resize (#655) On FreeBSD, if a UFS has trim: (-t) or MAC multilabel: (-l) flag, resize FS fail, because the _can_skip_ufs_resize check gets tripped up by the missing options. This was reported at FreeBSD Bugzilla: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=250496 and as LP: #1901958 Rather than fixing the parser as in the patches proposed there (and attempted in #636) this pull-request rips out all of it, and simplifies the code. We now use `growfs -N` and check if that returns an error. If it returns the correct kind of error, we can skip the resize, because we either are at the correct size, or the filesystem in question is broken or not UFS. If it returns the wrong kind of error, we just re-raise it. LP: #1901958 --- cloudinit/config/cc_resizefs.py | 68 +++++----------------- .../test_handler/test_handler_resizefs.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 978d2ee0..9afbb847 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -88,56 +85,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index db9a0414..28d55072 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3 From 5d4a9a4a50a496d27510f63217bcc0c25d9a8939 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 Nov 2020 14:12:54 -0500 Subject: add integration test for LP: #1900837 (#679) As the first test of this SRU cycle, this also introduces the sru_2020_11 mark to allow us to easily identify the set of tests generated for this SRU. --- tests/integration_tests/bugs/test_lp1900837.py | 28 ++++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 29 insertions(+) create mode 100644 tests/integration_tests/bugs/test_lp1900837.py diff --git a/tests/integration_tests/bugs/test_lp1900837.py b/tests/integration_tests/bugs/test_lp1900837.py new file mode 100644 index 00000000..3fe7d0d0 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1900837.py @@ -0,0 +1,28 @@ +"""Integration test for LP: #1900836. + +This test mirrors the reproducing steps from the reported bug: it changes the +permissions on cloud-init.log to 600 and confirms that they remain 600 after a +reboot. +""" +import pytest + + +def _get_log_perms(client): + return client.execute("stat -c %a /var/log/cloud-init.log") + + +@pytest.mark.sru_2020_11 +class TestLogPermissionsNotResetOnReboot: + def test_permissions_unchanged(self, client): + # Confirm that the current permissions aren't 600 + assert "644" == _get_log_perms(client) + + # Set permissions to 600 and confirm our assertion passes pre-reboot + client.execute("chmod 600 /var/log/cloud-init.log") + assert "600" == _get_log_perms(client) + + # Reboot + client.instance.restart() + + # Check that permissions are not reset on reboot + assert "600" == _get_log_perms(client) diff --git a/tox.ini b/tox.ini index c86d38e9..066f923a 100644 --- a/tox.ini +++ b/tox.ini @@ -169,3 +169,4 @@ markers = lxd_container: test will only run in LXD container user_data: the user data to be passed to the test instance instance_name: the name to be used for the test instance + sru_2020_11: test is part of the 2020/11 SRU verification -- cgit v1.2.3 From 66b4be8b6da188a0667bd8c86a25155b6f4f3f6c Mon Sep 17 00:00:00 2001 From: Jonathan Lung Date: Fri, 20 Nov 2020 15:59:51 -0500 Subject: Support configuring SSH host certificates. (#660) Existing config writes keys to /etc/ssh after deleting files matching a glob that includes certificate files. Since sshd looks for certificates in the same directory as the keys, a host certificate must be placed in this directory. This update enables the certificate's contents to be specified along with the keys. Co-authored-by: jonathan lung Co-authored-by: jonathan lung --- cloudinit/config/cc_ssh.py | 31 +++++++--- cloudinit/config/tests/test_ssh.py | 68 ++++++++++++++++++++-- .../modules/test_ssh_keys_provided.py | 13 +++++ tools/.github-cla-signers | 1 + 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 9b2a333a..05a16dbc 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -83,8 +83,9 @@ enabled by default. Host keys can be added using the ``ssh_keys`` configuration key. The argument to this config key should be a dictionary entries for the public and private keys of each desired key type. Entries in the ``ssh_keys`` config dict should -have keys in the format ``_private`` and ``_public``, -e.g. ``rsa_private: `` and ``rsa_public: ``. See below for supported +have keys in the format ``_private``, ``_public``, and, +optionally, ``_certificate``, e.g. ``rsa_private: ``, +``rsa_public: ``, and ``rsa_certificate: ``. See below for supported key types. Not all key types have to be specified, ones left unspecified will not be used. If this config option is used, then no keys will be generated. @@ -94,7 +95,8 @@ not be used. If this config option is used, then no keys will be generated. secure .. note:: - to specify multiline private host keys, use yaml multiline syntax + to specify multiline private host keys and certificates, use yaml + multiline syntax If no host keys are specified using ``ssh_keys``, then keys will be generated using ``ssh-keygen``. By default one public/private pair of each supported @@ -128,12 +130,17 @@ config flags are: ... -----END RSA PRIVATE KEY----- rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + rsa_certificate: | + ssh-rsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... dsa_private: | -----BEGIN DSA PRIVATE KEY----- MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qco ... -----END DSA PRIVATE KEY----- dsa_public: ssh-dsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + dsa_certificate: | + ssh-dsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... + ssh_genkeytypes: disable_root: disable_root_opts: @@ -169,6 +176,8 @@ for k in GENERATE_KEY_NAMES: CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) CONFIG_KEY_TO_FILE.update( {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + CONFIG_KEY_TO_FILE.update( + {"%s_certificate" % k: (KEY_FILE_TPL % k + "-cert.pub", 0o600)}) PRIV_TO_PUB["%s_private" % k] = "%s_public" % k KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' @@ -186,12 +195,18 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them + # if there are keys and/or certificates in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): - if key in CONFIG_KEY_TO_FILE: - tgt_fn = CONFIG_KEY_TO_FILE[key][0] - tgt_perms = CONFIG_KEY_TO_FILE[key][1] - util.write_file(tgt_fn, val, tgt_perms) + # skip entry if unrecognized + if key not in CONFIG_KEY_TO_FILE: + continue + tgt_fn = CONFIG_KEY_TO_FILE[key][0] + tgt_perms = CONFIG_KEY_TO_FILE[key][1] + util.write_file(tgt_fn, val, tgt_perms) + # set server to present the most recently identified certificate + if '_certificate' in key: + cert_config = {'HostCertificate': tgt_fn} + ssh_util.update_ssh_config(cert_config) for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: diff --git a/cloudinit/config/tests/test_ssh.py b/cloudinit/config/tests/test_ssh.py index 0c554414..87ccdb60 100644 --- a/cloudinit/config/tests/test_ssh.py +++ b/cloudinit/config/tests/test_ssh.py @@ -10,6 +10,8 @@ import logging LOG = logging.getLogger(__name__) MODPATH = "cloudinit.config.cc_ssh." +KEY_NAMES_NO_DSA = [name for name in cc_ssh.GENERATE_KEY_NAMES + if name not in 'dsa'] @mock.patch(MODPATH + "ssh_util.setup_user_keys") @@ -25,7 +27,7 @@ class TestHandleSsh(CiTestCase): } self.test_hostkey_files = [] hostkey_tmpdir = self.tmp_dir() - for key_type in ['dsa', 'ecdsa', 'ed25519', 'rsa']: + for key_type in cc_ssh.GENERATE_KEY_NAMES: key_data = self.test_hostkeys[key_type] filename = 'ssh_host_%s_key.pub' % key_type filepath = os.path.join(hostkey_tmpdir, filename) @@ -223,7 +225,7 @@ class TestHandleSsh(CiTestCase): cfg = {} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -252,7 +254,7 @@ class TestHandleSsh(CiTestCase): cfg = {'ssh_publish_hostkeys': {'enabled': True}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -339,7 +341,65 @@ class TestHandleSsh(CiTestCase): cfg = {'ssh_publish_hostkeys': {'enabled': True, 'blacklist': []}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['dsa', 'ecdsa', 'ed25519', 'rsa']] + in cc_ssh.GENERATE_KEY_NAMES] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) + + @mock.patch(MODPATH + "ug_util.normalize_users_groups") + @mock.patch(MODPATH + "util.write_file") + def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): + """Test handle with ssh keys and certificate.""" + # Populate a config dictionary to pass to handle() as well + # as the expected file-writing calls. + cfg = {"ssh_keys": {}} + + expected_calls = [] + for key_type in cc_ssh.GENERATE_KEY_NAMES: + private_name = "{}_private".format(key_type) + public_name = "{}_public".format(key_type) + cert_name = "{}_certificate".format(key_type) + + # Actual key contents don"t have to be realistic + private_value = "{}_PRIVATE_KEY".format(key_type) + public_value = "{}_PUBLIC_KEY".format(key_type) + cert_value = "{}_CERT_KEY".format(key_type) + + cfg["ssh_keys"][private_name] = private_value + cfg["ssh_keys"][public_name] = public_value + cfg["ssh_keys"][cert_name] = cert_value + + expected_calls.extend([ + mock.call( + '/etc/ssh/ssh_host_{}_key'.format(key_type), + private_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key.pub'.format(key_type), + public_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key-cert.pub'.format(key_type), + cert_value, + 384 + ), + mock.call( + '/etc/ssh/sshd_config', + ('HostCertificate /etc/ssh/ssh_host_{}_key-cert.pub' + '\n'.format(key_type)), + preserve_mode=True + ) + ]) + + # Run the handler. + m_nug.return_value = ([], {}) + with mock.patch(MODPATH + 'ssh_util.parse_ssh_config', + return_value=[]): + cc_ssh.handle("name", cfg, self.tmp_cloud(distro='ubuntu'), + LOG, None) + + # Check that all expected output has been done. + for call_ in expected_calls: + self.assertIn(call_, m_write_file.call_args_list) diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py index dc6d2fc1..27d193c1 100644 --- a/tests/integration_tests/modules/test_ssh_keys_provided.py +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -45,6 +45,7 @@ ssh_keys: A3tFPEOxauXpzCt8f8eXsz0WQXAgIKW2h8zu5QHjomioU3i27mtE -----END RSA PRIVATE KEY----- rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97N root@xenial-lxd + rsa_certificate: ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpgBP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97NAAAAAAAAAAAAAAACAAAACnhlbmlhbC1seGQAAAAAAAAAAF+vVEIAAAAAYY83bgAAAAAAAAAAAAAAAAAAADMAAAALc3NoLWVkMjU1MTkAAAAgz4SlDwbq53ZrRsnS6ISdwxgFDRpnEX44K8jFmLpI9NAAAABTAAAAC3NzaC1lZDI1NTE5AAAAQMWpiRWKNMFvRX0g6OQOELMqDhtNBpkIN92IyO25qiY2oDSd1NyVme6XnGDFt8CS7z5NufV04doP4aacLOBbQww= root@xenial-lxd dsa_private: | -----BEGIN DSA PRIVATE KEY----- MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXrhOVAfzZ6+jklP @@ -108,6 +109,18 @@ class TestSshKeysProvided: "4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un" "RQvLZpMRdywBm") in out + def test_ssh_rsa_certificate_provided(self, class_client): + """Test rsa certificate was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key-cert.pub") + assert ( + "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpg" + "BP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAAD") in out + + def test_ssh_certificate_updated_sshd_config(self, class_client): + """Test ssh certificate was added to /etc/ssh/sshd_config.""" + out = class_client.read_from_file("/etc/ssh/sshd_config").strip() + assert "HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub" in out + def test_ssh_ecdsa_keys_provided(self, class_client): """Test ecdsa public key was imported.""" out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key.pub") diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b2464768..9b594a44 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -15,6 +15,7 @@ jqueuniet jsf9k landon912 lucasmoura +lungj manuelisimo marlluslustosa matthewruffell -- cgit v1.2.3 From a4d0feb050e32277a218e45bfb6a496d26ff46d0 Mon Sep 17 00:00:00 2001 From: aswinrajamannar <39812128+aswinrajamannar@users.noreply.github.com> Date: Mon, 23 Nov 2020 07:04:05 -0800 Subject: Ability to hot-attach NICs to preprovisioned VMs before reprovisioning (#613) Adds the ability to run the Azure preprovisioned VMs as NIC-less and then hot-attach them when assigned for reprovision. The NIC on the preprovisioned VM is hot-detached as soon as it reports ready and goes into wait for one or more interfaces to be hot-attached. Once they are attached, cloud-init gets the expected number of NICs (in case there are more than one) that will be attached from IMDS and waits until all of them are attached. After all the NICs are attached, reprovision proceeds as usual. --- cloudinit/distros/networking.py | 15 + cloudinit/distros/tests/test_networking.py | 31 ++ cloudinit/sources/DataSourceAzure.py | 515 ++++++++++++++++++++++-- cloudinit/sources/helpers/netlink.py | 102 ++++- cloudinit/sources/helpers/tests/test_netlink.py | 74 +++- tests/unittests/test_datasource/test_azure.py | 436 +++++++++++++++++++- tools/.github-cla-signers | 1 + 7 files changed, 1109 insertions(+), 65 deletions(-) diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index e407fa29..c291196a 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -2,6 +2,7 @@ import abc import logging import os +from cloudinit import subp from cloudinit import net, util @@ -175,6 +176,10 @@ class Networking(metaclass=abc.ABCMeta): if strict: raise RuntimeError(msg) + @abc.abstractmethod + def try_set_link_up(self, devname: DeviceName) -> bool: + """Try setting the link to up explicitly and return if it is up.""" + class BSDNetworking(Networking): """Implementation of networking functionality shared across BSDs.""" @@ -185,6 +190,9 @@ class BSDNetworking(Networking): def settle(self, *, exists=None) -> None: """BSD has no equivalent to `udevadm settle`; noop.""" + def try_set_link_up(self, devname: DeviceName) -> bool: + raise NotImplementedError() + class LinuxNetworking(Networking): """Implementation of networking functionality common to Linux distros.""" @@ -214,3 +222,10 @@ class LinuxNetworking(Networking): if exists is not None: exists = net.sys_dev_path(exists) util.udevadm_settle(exists=exists) + + def try_set_link_up(self, devname: DeviceName) -> bool: + """Try setting the link to up explicitly and return if it is up. + Not guaranteed to bring the interface up. The caller is expected to + add wait times before retrying.""" + subp.subp(['ip', 'link', 'set', devname, 'up']) + return self.is_up(devname) diff --git a/cloudinit/distros/tests/test_networking.py b/cloudinit/distros/tests/test_networking.py index b9a63842..ec508f4d 100644 --- a/cloudinit/distros/tests/test_networking.py +++ b/cloudinit/distros/tests/test_networking.py @@ -30,6 +30,9 @@ def generic_networking_cls(): def settle(self, *args, **kwargs): raise NotImplementedError + def try_set_link_up(self, *args, **kwargs): + raise NotImplementedError + error = AssertionError("Unexpectedly used /sys in generic networking code") with mock.patch( "cloudinit.net.get_sys_class_path", side_effect=error, @@ -74,6 +77,34 @@ class TestLinuxNetworkingIsPhysical: assert LinuxNetworking().is_physical(devname) +class TestBSDNetworkingTrySetLinkUp: + def test_raises_notimplementederror(self): + with pytest.raises(NotImplementedError): + BSDNetworking().try_set_link_up("eth0") + + +@mock.patch("cloudinit.net.is_up") +@mock.patch("cloudinit.distros.networking.subp.subp") +class TestLinuxNetworkingTrySetLinkUp: + def test_calls_subp_return_true(self, m_subp, m_is_up): + devname = "eth0" + m_is_up.return_value = True + is_success = LinuxNetworking().try_set_link_up(devname) + + assert (mock.call(['ip', 'link', 'set', devname, 'up']) == + m_subp.call_args_list[-1]) + assert is_success + + def test_calls_subp_return_false(self, m_subp, m_is_up): + devname = "eth0" + m_is_up.return_value = False + is_success = LinuxNetworking().try_set_link_up(devname) + + assert (mock.call(['ip', 'link', 'set', devname, 'up']) == + m_subp.call_args_list[-1]) + assert not is_success + + class TestBSDNetworkingSettle: def test_settle_doesnt_error(self): # This also implicitly tests that it doesn't use subp.subp diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f777a007..04ff2131 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -12,8 +12,10 @@ import os import os.path import re from time import time +from time import sleep from xml.dom import minidom import xml.etree.ElementTree as ET +from enum import Enum from cloudinit import dmi from cloudinit import log as logging @@ -67,13 +69,27 @@ DEFAULT_FS = 'ext4' # DMI chassis-asset-tag is set static for all azure instances AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77' REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds" +REPROVISION_NIC_ATTACH_MARKER_FILE = "/var/lib/cloud/data/wait_for_nic_attach" +REPROVISION_NIC_DETACHED_MARKER_FILE = "/var/lib/cloud/data/nic_detached" REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready" AGENT_SEED_DIR = '/var/lib/waagent' + # In the event where the IMDS primary server is not # available, it takes 1s to fallback to the secondary one IMDS_TIMEOUT_IN_SECONDS = 2 IMDS_URL = "http://169.254.169.254/metadata/" +IMDS_VER = "2019-06-01" +IMDS_VER_PARAM = "api-version={}".format(IMDS_VER) + + +class metadata_type(Enum): + compute = "{}instance?{}".format(IMDS_URL, IMDS_VER_PARAM) + network = "{}instance/network?{}".format(IMDS_URL, + IMDS_VER_PARAM) + reprovisiondata = "{}reprovisiondata?{}".format(IMDS_URL, + IMDS_VER_PARAM) + PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0" @@ -434,20 +450,39 @@ class DataSourceAzure(sources.DataSource): # need to look in the datadir and consider that valid ddir = self.ds_cfg['data_dir'] + # The order in which the candidates are inserted matters here, because + # it determines the value of ret. More specifically, the first one in + # the candidate list determines the path to take in order to get the + # metadata we need. candidates = [self.seed_dir] if os.path.isfile(REPROVISION_MARKER_FILE): candidates.insert(0, "IMDS") + report_diagnostic_event("Reprovision marker file already present " + "before crawling Azure metadata: %s" % + REPROVISION_MARKER_FILE, + logger_func=LOG.debug) + elif os.path.isfile(REPROVISION_NIC_ATTACH_MARKER_FILE): + candidates.insert(0, "NIC_ATTACH_MARKER_PRESENT") + report_diagnostic_event("Reprovision nic attach marker file " + "already present before crawling Azure " + "metadata: %s" % + REPROVISION_NIC_ATTACH_MARKER_FILE, + logger_func=LOG.debug) candidates.extend(list_possible_azure_ds_devs()) if ddir: candidates.append(ddir) found = None reprovision = False + reprovision_after_nic_attach = False for cdev in candidates: try: if cdev == "IMDS": ret = None reprovision = True + elif cdev == "NIC_ATTACH_MARKER_PRESENT": + ret = None + reprovision_after_nic_attach = True elif cdev.startswith("/dev/"): if util.is_FreeBSD(): ret = util.mount_cb(cdev, load_azure_ds_dir, @@ -472,12 +507,19 @@ class DataSourceAzure(sources.DataSource): continue perform_reprovision = reprovision or self._should_reprovision(ret) - if perform_reprovision: + perform_reprovision_after_nic_attach = ( + reprovision_after_nic_attach or + self._should_reprovision_after_nic_attach(ret)) + + if perform_reprovision or perform_reprovision_after_nic_attach: if util.is_FreeBSD(): msg = "Free BSD is not supported for PPS VMs" report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) + if perform_reprovision_after_nic_attach: + self._wait_for_all_nics_ready() ret = self._reprovision() + imds_md = get_metadata_from_imds( self.fallback_interface, retries=10) (md, userdata_raw, cfg, files) = ret @@ -508,7 +550,7 @@ class DataSourceAzure(sources.DataSource): crawled_data['metadata']['random_seed'] = seed crawled_data['metadata']['instance-id'] = self._iid() - if perform_reprovision: + if perform_reprovision or perform_reprovision_after_nic_attach: LOG.info("Reporting ready to Azure after getting ReprovisionData") use_cached_ephemeral = ( self.distro.networking.is_up(self.fallback_interface) and @@ -617,14 +659,12 @@ class DataSourceAzure(sources.DataSource): LOG.debug('Retrieved SSH keys from IMDS') except KeyError: log_msg = 'Unable to get keys from IMDS, falling back to OVF' - LOG.debug(log_msg) report_diagnostic_event(log_msg, logger_func=LOG.debug) try: ssh_keys = self.metadata['public-keys'] LOG.debug('Retrieved keys from OVF') except KeyError: log_msg = 'No keys available from OVF' - LOG.debug(log_msg) report_diagnostic_event(log_msg, logger_func=LOG.debug) return ssh_keys @@ -660,10 +700,293 @@ class DataSourceAzure(sources.DataSource): LOG.debug("negotiating already done for %s", self.get_instance_id()) + @azure_ds_telemetry_reporter + def _wait_for_nic_detach(self, nl_sock): + """Use the netlink socket provided to wait for nic detach event. + NOTE: The function doesn't close the socket. The caller owns closing + the socket and disposing it safely. + """ + try: + ifname = None + + # Preprovisioned VM will only have one NIC, and it gets + # detached immediately after deployment. + with events.ReportEventStack( + name="wait-for-nic-detach", + description=("wait for nic detach"), + parent=azure_ds_reporter): + ifname = netlink.wait_for_nic_detach_event(nl_sock) + if ifname is None: + msg = ("Preprovisioned nic not detached as expected. " + "Proceeding without failing.") + report_diagnostic_event(msg, logger_func=LOG.warning) + else: + report_diagnostic_event("The preprovisioned nic %s is detached" + % ifname, logger_func=LOG.warning) + path = REPROVISION_NIC_DETACHED_MARKER_FILE + LOG.info("Creating a marker file for nic detached: %s", path) + util.write_file(path, "{pid}: {time}\n".format( + pid=os.getpid(), time=time())) + except AssertionError as error: + report_diagnostic_event(error, logger_func=LOG.error) + raise + + @azure_ds_telemetry_reporter + def wait_for_link_up(self, ifname): + """In cases where the link state is still showing down after a nic is + hot-attached, we can attempt to bring it up by forcing the hv_netvsc + drivers to query the link state by unbinding and then binding the + device. This function attempts infinitely until the link is up, + because we cannot proceed further until we have a stable link.""" + + if self.distro.networking.try_set_link_up(ifname): + report_diagnostic_event("The link %s is already up." % ifname, + logger_func=LOG.info) + return + + LOG.info("Attempting to bring %s up", ifname) + + attempts = 0 + while True: + + LOG.info("Unbinding and binding the interface %s", ifname) + devicename = net.read_sys_net(ifname, + 'device/device_id').strip('{}') + util.write_file('/sys/bus/vmbus/drivers/hv_netvsc/unbind', + devicename) + util.write_file('/sys/bus/vmbus/drivers/hv_netvsc/bind', + devicename) + + attempts = attempts + 1 + if self.distro.networking.try_set_link_up(ifname): + msg = "The link %s is up after %s attempts" % (ifname, + attempts) + report_diagnostic_event(msg, logger_func=LOG.info) + return + + sleep_duration = 1 + msg = ("Link is not up after %d attempts with %d seconds sleep " + "between attempts." % (attempts, sleep_duration)) + + if attempts % 10 == 0: + report_diagnostic_event(msg, logger_func=LOG.info) + else: + LOG.info(msg) + + sleep(sleep_duration) + + @azure_ds_telemetry_reporter + def _create_report_ready_marker(self): + path = REPORTED_READY_MARKER_FILE + LOG.info( + "Creating a marker file to report ready: %s", path) + util.write_file(path, "{pid}: {time}\n".format( + pid=os.getpid(), time=time())) + report_diagnostic_event( + 'Successfully created reported ready marker file ' + 'while in the preprovisioning pool.', + logger_func=LOG.debug) + + @azure_ds_telemetry_reporter + def _report_ready_if_needed(self): + """Report ready to the platform if the marker file is not present, + and create the marker file. + """ + have_not_reported_ready = ( + not os.path.isfile(REPORTED_READY_MARKER_FILE)) + + if have_not_reported_ready: + report_diagnostic_event("Reporting ready before nic detach", + logger_func=LOG.info) + try: + with EphemeralDHCPv4WithReporting(azure_ds_reporter) as lease: + self._report_ready(lease=lease) + except Exception as e: + report_diagnostic_event("Exception reporting ready during " + "preprovisioning before nic detach: %s" + % e, logger_func=LOG.error) + raise + self._create_report_ready_marker() + else: + report_diagnostic_event("Already reported ready before nic detach." + " The marker file already exists: %s" % + REPORTED_READY_MARKER_FILE, + logger_func=LOG.error) + + @azure_ds_telemetry_reporter + def _check_if_nic_is_primary(self, ifname): + """Check if a given interface is the primary nic or not. If it is the + primary nic, then we also get the expected total nic count from IMDS. + IMDS will process the request and send a response only for primary NIC. + """ + is_primary = False + expected_nic_count = -1 + imds_md = None + + # For now, only a VM's primary NIC can contact IMDS and WireServer. If + # DHCP fails for a NIC, we have no mechanism to determine if the NIC is + # primary or secondary. In this case, the desired behavior is to fail + # VM provisioning if there is any DHCP failure when trying to determine + # the primary NIC. + try: + with events.ReportEventStack( + name="obtain-dhcp-lease", + description=("obtain dhcp lease for %s when attempting to " + "determine primary NIC during reprovision of " + "a pre-provisioned VM" % ifname), + parent=azure_ds_reporter): + dhcp_ctx = EphemeralDHCPv4( + iface=ifname, + dhcp_log_func=dhcp_log_cb) + dhcp_ctx.obtain_lease() + except Exception as e: + report_diagnostic_event("Giving up. Failed to obtain dhcp lease " + "for %s when attempting to determine " + "primary NIC during reprovision due to %s" + % (ifname, e), logger_func=LOG.error) + raise + + # Primary nic detection will be optimized in the future. The fact that + # primary nic is being attached first helps here. Otherwise each nic + # could add several seconds of delay. + try: + imds_md = get_metadata_from_imds( + ifname, + 5, + metadata_type.network) + except Exception as e: + LOG.warning( + "Failed to get network metadata using nic %s. Attempt to " + "contact IMDS failed with error %s. Assuming this is not the " + "primary nic.", ifname, e) + finally: + # If we are not the primary nic, then clean the dhcp context. + if imds_md is None: + dhcp_ctx.clean_network() + + if imds_md is not None: + # Only primary NIC will get a response from IMDS. + LOG.info("%s is the primary nic", ifname) + is_primary = True + + # If primary, set ephemeral dhcp ctx so we can report ready + self._ephemeral_dhcp_ctx = dhcp_ctx + + # Set the expected nic count based on the response received. + expected_nic_count = len( + imds_md['interface']) + report_diagnostic_event("Expected nic count: %d" % + expected_nic_count, logger_func=LOG.info) + + return is_primary, expected_nic_count + + @azure_ds_telemetry_reporter + def _wait_for_hot_attached_nics(self, nl_sock): + """Wait until all the expected nics for the vm are hot-attached. + The expected nic count is obtained by requesting the network metadata + from IMDS. + """ + LOG.info("Waiting for nics to be hot-attached") + try: + # Wait for nics to be attached one at a time, until we know for + # sure that all nics have been attached. + nics_found = [] + primary_nic_found = False + expected_nic_count = -1 + + # Wait for netlink nic attach events. After the first nic is + # attached, we are already in the customer vm deployment path and + # so eerything from then on should happen fast and avoid + # unnecessary delays wherever possible. + while True: + ifname = None + with events.ReportEventStack( + name="wait-for-nic-attach", + description=("wait for nic attach after %d nics have " + "been attached" % len(nics_found)), + parent=azure_ds_reporter): + ifname = netlink.wait_for_nic_attach_event(nl_sock, + nics_found) + + # wait_for_nic_attach_event guarantees that ifname it not None + nics_found.append(ifname) + report_diagnostic_event("Detected nic %s attached." % ifname, + logger_func=LOG.info) + + # Attempt to bring the interface's operating state to + # UP in case it is not already. + self.wait_for_link_up(ifname) + + # If primary nic is not found, check if this is it. The + # platform will attach the primary nic first so we + # won't be in primary_nic_found = false state for long. + if not primary_nic_found: + LOG.info("Checking if %s is the primary nic", + ifname) + (primary_nic_found, expected_nic_count) = ( + self._check_if_nic_is_primary(ifname)) + + # Exit criteria: check if we've discovered all nics + if (expected_nic_count != -1 + and len(nics_found) >= expected_nic_count): + LOG.info("Found all the nics for this VM.") + break + + except AssertionError as error: + report_diagnostic_event(error, logger_func=LOG.error) + + @azure_ds_telemetry_reporter + def _wait_for_all_nics_ready(self): + """Wait for nic(s) to be hot-attached. There may be multiple nics + depending on the customer request. + But only primary nic would be able to communicate with wireserver + and IMDS. So we detect and save the primary nic to be used later. + """ + + nl_sock = None + try: + nl_sock = netlink.create_bound_netlink_socket() + + report_ready_marker_present = bool( + os.path.isfile(REPORTED_READY_MARKER_FILE)) + + # Report ready if the marker file is not already present. + # The nic of the preprovisioned vm gets hot-detached as soon as + # we report ready. So no need to save the dhcp context. + self._report_ready_if_needed() + + has_nic_been_detached = bool( + os.path.isfile(REPROVISION_NIC_DETACHED_MARKER_FILE)) + + if not has_nic_been_detached: + LOG.info("NIC has not been detached yet.") + self._wait_for_nic_detach(nl_sock) + + # If we know that the preprovisioned nic has been detached, and we + # still have a fallback nic, then it means the VM must have + # rebooted as part of customer assignment, and all the nics have + # already been attached by the Azure platform. So there is no need + # to wait for nics to be hot-attached. + if not self.fallback_interface: + self._wait_for_hot_attached_nics(nl_sock) + else: + report_diagnostic_event("Skipping waiting for nic attach " + "because we already have a fallback " + "interface. Report Ready marker " + "present before detaching nics: %s" % + report_ready_marker_present, + logger_func=LOG.info) + except netlink.NetlinkCreateSocketError as e: + report_diagnostic_event(e, logger_func=LOG.warning) + raise + finally: + if nl_sock: + nl_sock.close() + def _poll_imds(self): """Poll IMDS for the new provisioning data until we get a valid response. Then return the returned JSON object.""" - url = IMDS_URL + "reprovisiondata?api-version=2017-04-02" + url = metadata_type.reprovisiondata.value headers = {"Metadata": "true"} nl_sock = None report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE)) @@ -705,17 +1028,31 @@ class DataSourceAzure(sources.DataSource): logger_func=LOG.warning) return False - LOG.debug("Wait for vnetswitch to happen") + # When the interface is hot-attached, we would have already + # done dhcp and set the dhcp context. In that case, skip + # the attempt to do dhcp. + is_ephemeral_ctx_present = self._ephemeral_dhcp_ctx is not None + msg = ("Unexpected error. Dhcp context is not expected to be already " + "set when we need to wait for vnet switch") + if is_ephemeral_ctx_present and report_ready: + report_diagnostic_event(msg, logger_func=LOG.error) + raise RuntimeError(msg) + while True: try: - # Save our EphemeralDHCPv4 context to avoid repeated dhcp - with events.ReportEventStack( - name="obtain-dhcp-lease", - description="obtain dhcp lease", - parent=azure_ds_reporter): - self._ephemeral_dhcp_ctx = EphemeralDHCPv4( - dhcp_log_func=dhcp_log_cb) - lease = self._ephemeral_dhcp_ctx.obtain_lease() + # Since is_ephemeral_ctx_present is set only once, this ensures + # that with regular reprovisioning, dhcp is always done every + # time the loop runs. + if not is_ephemeral_ctx_present: + # Save our EphemeralDHCPv4 context to avoid repeated dhcp + # later when we report ready + with events.ReportEventStack( + name="obtain-dhcp-lease", + description="obtain dhcp lease", + parent=azure_ds_reporter): + self._ephemeral_dhcp_ctx = EphemeralDHCPv4( + dhcp_log_func=dhcp_log_cb) + lease = self._ephemeral_dhcp_ctx.obtain_lease() if vnet_switched: dhcp_attempts += 1 @@ -737,17 +1074,10 @@ class DataSourceAzure(sources.DataSource): self._ephemeral_dhcp_ctx.clean_network() raise sources.InvalidMetaDataException(msg) - path = REPORTED_READY_MARKER_FILE - LOG.info( - "Creating a marker file to report ready: %s", path) - util.write_file(path, "{pid}: {time}\n".format( - pid=os.getpid(), time=time())) - report_diagnostic_event( - 'Successfully created reported ready marker file ' - 'while in the preprovisioning pool.', - logger_func=LOG.debug) + self._create_report_ready_marker() report_ready = False + LOG.debug("Wait for vnetswitch to happen") with events.ReportEventStack( name="wait-for-media-disconnect-connect", description="wait for vnet switch", @@ -863,6 +1193,35 @@ class DataSourceAzure(sources.DataSource): "connectivity issues: %s" % e, logger_func=LOG.warning) return False + def _should_reprovision_after_nic_attach(self, candidate_metadata) -> bool: + """Whether or not we should wait for nic attach and then poll + IMDS for reprovisioning data. Also sets a marker file to poll IMDS. + + The marker file is used for the following scenario: the VM boots into + wait for nic attach, which we expect to be proceeding infinitely until + the nic is attached. If for whatever reason the platform moves us to a + new host (for instance a hardware issue), we need to keep waiting. + However, since the VM reports ready to the Fabric, we will not attach + the ISO, thus cloud-init needs to have a way of knowing that it should + jump back into the waiting mode in order to retrieve the ovf_env. + + @param candidate_metadata: Metadata obtained from reading ovf-env. + @return: Whether to reprovision after waiting for nics to be attached. + """ + if not candidate_metadata: + return False + (_md, _userdata_raw, cfg, _files) = candidate_metadata + path = REPROVISION_NIC_ATTACH_MARKER_FILE + if (cfg.get('PreprovisionedVMType', None) == "Savable" or + os.path.isfile(path)): + if not os.path.isfile(path): + LOG.info("Creating a marker file to wait for nic attach: %s", + path) + util.write_file(path, "{pid}: {time}\n".format( + pid=os.getpid(), time=time())) + return True + return False + def _should_reprovision(self, ret): """Whether or not we should poll IMDS for reprovisioning data. Also sets a marker file to poll IMDS. @@ -879,6 +1238,7 @@ class DataSourceAzure(sources.DataSource): (_md, _userdata_raw, cfg, _files) = ret path = REPROVISION_MARKER_FILE if (cfg.get('PreprovisionedVm') is True or + cfg.get('PreprovisionedVMType', None) == 'Running' or os.path.isfile(path)): if not os.path.isfile(path): LOG.info("Creating a marker file to poll imds: %s", @@ -944,6 +1304,8 @@ class DataSourceAzure(sources.DataSource): util.del_file(REPORTED_READY_MARKER_FILE) util.del_file(REPROVISION_MARKER_FILE) + util.del_file(REPROVISION_NIC_ATTACH_MARKER_FILE) + util.del_file(REPROVISION_NIC_DETACHED_MARKER_FILE) return fabric_data @azure_ds_telemetry_reporter @@ -1399,34 +1761,109 @@ def read_azure_ovf(contents): if 'ssh_pwauth' not in cfg and password: cfg['ssh_pwauth'] = True - cfg['PreprovisionedVm'] = _extract_preprovisioned_vm_setting(dom) + preprovisioning_cfg = _get_preprovisioning_cfgs(dom) + cfg = util.mergemanydict([cfg, preprovisioning_cfg]) return (md, ud, cfg) @azure_ds_telemetry_reporter -def _extract_preprovisioned_vm_setting(dom): - """Read the preprovision flag from the ovf. It should not - exist unless true.""" +def _get_preprovisioning_cfgs(dom): + """Read the preprovisioning related flags from ovf and populates a dict + with the info. + + Two flags are in use today: PreprovisionedVm bool and + PreprovisionedVMType enum. In the long term, the PreprovisionedVm bool + will be deprecated in favor of PreprovisionedVMType string/enum. + + Only these combinations of values are possible today: + - PreprovisionedVm=True and PreprovisionedVMType=Running + - PreprovisionedVm=False and PreprovisionedVMType=Savable + - PreprovisionedVm is missing and PreprovisionedVMType=Running/Savable + - PreprovisionedVm=False and PreprovisionedVMType is missing + + More specifically, this will never happen: + - PreprovisionedVm=True and PreprovisionedVMType=Savable + """ + cfg = { + "PreprovisionedVm": False, + "PreprovisionedVMType": None + } + platform_settings_section = find_child( dom.documentElement, lambda n: n.localName == "PlatformSettingsSection") if not platform_settings_section or len(platform_settings_section) == 0: LOG.debug("PlatformSettingsSection not found") - return False + return cfg platform_settings = find_child( platform_settings_section[0], lambda n: n.localName == "PlatformSettings") if not platform_settings or len(platform_settings) == 0: LOG.debug("PlatformSettings not found") - return False - preprovisionedVm = find_child( + return cfg + + # Read the PreprovisionedVm bool flag. This should be deprecated when the + # platform has removed PreprovisionedVm and only surfaces + # PreprovisionedVMType. + cfg["PreprovisionedVm"] = _get_preprovisionedvm_cfg_value( + platform_settings) + + cfg["PreprovisionedVMType"] = _get_preprovisionedvmtype_cfg_value( + platform_settings) + return cfg + + +@azure_ds_telemetry_reporter +def _get_preprovisionedvm_cfg_value(platform_settings): + preprovisionedVm = False + + # Read the PreprovisionedVm bool flag. This should be deprecated when the + # platform has removed PreprovisionedVm and only surfaces + # PreprovisionedVMType. + preprovisionedVmVal = find_child( platform_settings[0], lambda n: n.localName == "PreprovisionedVm") - if not preprovisionedVm or len(preprovisionedVm) == 0: + if not preprovisionedVmVal or len(preprovisionedVmVal) == 0: LOG.debug("PreprovisionedVm not found") - return False - return util.translate_bool(preprovisionedVm[0].firstChild.nodeValue) + return preprovisionedVm + preprovisionedVm = util.translate_bool( + preprovisionedVmVal[0].firstChild.nodeValue) + + report_diagnostic_event( + "PreprovisionedVm: %s" % preprovisionedVm, logger_func=LOG.info) + + return preprovisionedVm + + +@azure_ds_telemetry_reporter +def _get_preprovisionedvmtype_cfg_value(platform_settings): + preprovisionedVMType = None + + # Read the PreprovisionedVMType value from the ovf. It can be + # 'Running' or 'Savable' or not exist. This enum value is intended to + # replace PreprovisionedVm bool flag in the long term. + # A Running VM is the same as preprovisioned VMs of today. This is + # equivalent to having PreprovisionedVm=True. + # A Savable VM is one whose nic is hot-detached immediately after it + # reports ready the first time to free up the network resources. + # Once assigned to customer, the customer-requested nics are + # hot-attached to it and reprovision happens like today. + preprovisionedVMTypeVal = find_child( + platform_settings[0], + lambda n: n.localName == "PreprovisionedVMType") + if (not preprovisionedVMTypeVal or len(preprovisionedVMTypeVal) == 0 or + preprovisionedVMTypeVal[0].firstChild is None): + LOG.debug("PreprovisionedVMType not found") + return preprovisionedVMType + + preprovisionedVMType = preprovisionedVMTypeVal[0].firstChild.nodeValue + + report_diagnostic_event( + "PreprovisionedVMType: %s" % preprovisionedVMType, + logger_func=LOG.info) + + return preprovisionedVMType def encrypt_pass(password, salt_id="$6$"): @@ -1588,8 +2025,10 @@ def _generate_network_config_from_fallback_config() -> dict: @azure_ds_telemetry_reporter -def get_metadata_from_imds(fallback_nic, retries): - """Query Azure's network metadata service, returning a dictionary. +def get_metadata_from_imds(fallback_nic, + retries, + md_type=metadata_type.compute): + """Query Azure's instance metadata service, returning a dictionary. If network is not up, setup ephemeral dhcp on fallback_nic to talk to the IMDS. For more info on IMDS: @@ -1604,7 +2043,7 @@ def get_metadata_from_imds(fallback_nic, retries): """ kwargs = {'logfunc': LOG.debug, 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)', - 'func': _get_metadata_from_imds, 'args': (retries,)} + 'func': _get_metadata_from_imds, 'args': (retries, md_type,)} if net.is_up(fallback_nic): return util.log_time(**kwargs) else: @@ -1620,9 +2059,9 @@ def get_metadata_from_imds(fallback_nic, retries): @azure_ds_telemetry_reporter -def _get_metadata_from_imds(retries): +def _get_metadata_from_imds(retries, md_type=metadata_type.compute): - url = IMDS_URL + "instance?api-version=2019-06-01" + url = md_type.value headers = {"Metadata": "true"} try: response = readurl( diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index c2ad587b..e13d6834 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -185,6 +185,54 @@ def read_rta_oper_state(data): return InterfaceOperstate(ifname, operstate) +def wait_for_nic_attach_event(netlink_socket, existing_nics): + '''Block until a single nic is attached. + + :param: netlink_socket: netlink_socket to receive events + :param: existing_nics: List of existing nics so that we can skip them. + :raises: AssertionError if netlink_socket is none. + ''' + LOG.debug("Preparing to wait for nic attach.") + ifname = None + + def should_continue_cb(iname, carrier, prevCarrier): + if iname in existing_nics: + return True + nonlocal ifname + ifname = iname + return False + + # We can return even if the operational state of the new nic is DOWN + # because we set it to UP before doing dhcp. + read_netlink_messages(netlink_socket, + None, + [RTM_NEWLINK], + [OPER_UP, OPER_DOWN], + should_continue_cb) + return ifname + + +def wait_for_nic_detach_event(netlink_socket): + '''Block until a single nic is detached and its operational state is down. + + :param: netlink_socket: netlink_socket to receive events. + ''' + LOG.debug("Preparing to wait for nic detach.") + ifname = None + + def should_continue_cb(iname, carrier, prevCarrier): + nonlocal ifname + ifname = iname + return False + + read_netlink_messages(netlink_socket, + None, + [RTM_DELLINK], + [OPER_DOWN], + should_continue_cb) + return ifname + + def wait_for_media_disconnect_connect(netlink_socket, ifname): '''Block until media disconnect and connect has happened on an interface. Listens on netlink socket to receive netlink events and when the carrier @@ -198,10 +246,42 @@ def wait_for_media_disconnect_connect(netlink_socket, ifname): assert (netlink_socket is not None), ("netlink socket is none") assert (ifname is not None), ("interface name is none") assert (len(ifname) > 0), ("interface name cannot be empty") + + def should_continue_cb(iname, carrier, prevCarrier): + # check for carrier down, up sequence + isVnetSwitch = (prevCarrier == OPER_DOWN) and (carrier == OPER_UP) + if isVnetSwitch: + LOG.debug("Media switch happened on %s.", ifname) + return False + return True + + LOG.debug("Wait for media disconnect and reconnect to happen") + read_netlink_messages(netlink_socket, + ifname, + [RTM_NEWLINK, RTM_DELLINK], + [OPER_UP, OPER_DOWN], + should_continue_cb) + + +def read_netlink_messages(netlink_socket, + ifname_filter, + rtm_types, + operstates, + should_continue_callback): + ''' Reads from the netlink socket until the condition specified by + the continuation callback is met. + + :param: netlink_socket: netlink_socket to receive events. + :param: ifname_filter: if not None, will only listen for this interface. + :param: rtm_types: Type of netlink events to listen for. + :param: operstates: Operational states to listen. + :param: should_continue_callback: Specifies when to stop listening. + ''' + if netlink_socket is None: + raise RuntimeError("Netlink socket is none") + data = bytes() carrier = OPER_UP prevCarrier = OPER_UP - data = bytes() - LOG.debug("Wait for media disconnect and reconnect to happen") while True: recv_data = read_netlink_socket(netlink_socket, SELECT_TIMEOUT) if recv_data is None: @@ -223,26 +303,26 @@ def wait_for_media_disconnect_connect(netlink_socket, ifname): padlen = (nlheader.length+PAD_ALIGNMENT-1) & ~(PAD_ALIGNMENT-1) offset = offset + padlen LOG.debug('offset to next netlink message: %d', offset) - # Ignore any messages not new link or del link - if nlheader.type not in [RTM_NEWLINK, RTM_DELLINK]: + # Continue if we are not interested in this message. + if nlheader.type not in rtm_types: continue interface_state = read_rta_oper_state(nl_msg) if interface_state is None: LOG.debug('Failed to read rta attributes: %s', interface_state) continue - if interface_state.ifname != ifname: + if (ifname_filter is not None and + interface_state.ifname != ifname_filter): LOG.debug( "Ignored netlink event on interface %s. Waiting for %s.", - interface_state.ifname, ifname) + interface_state.ifname, ifname_filter) continue - if interface_state.operstate not in [OPER_UP, OPER_DOWN]: + if interface_state.operstate not in operstates: continue prevCarrier = carrier carrier = interface_state.operstate - # check for carrier down, up sequence - isVnetSwitch = (prevCarrier == OPER_DOWN) and (carrier == OPER_UP) - if isVnetSwitch: - LOG.debug("Media switch happened on %s.", ifname) + if not should_continue_callback(interface_state.ifname, + carrier, + prevCarrier): return data = data[offset:] diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py index 10760bd6..cafe3961 100644 --- a/cloudinit/sources/helpers/tests/test_netlink.py +++ b/cloudinit/sources/helpers/tests/test_netlink.py @@ -9,9 +9,10 @@ import codecs from cloudinit.sources.helpers.netlink import ( NetlinkCreateSocketError, create_bound_netlink_socket, read_netlink_socket, read_rta_oper_state, unpack_rta_attr, wait_for_media_disconnect_connect, + wait_for_nic_attach_event, wait_for_nic_detach_event, OPER_DOWN, OPER_UP, OPER_DORMANT, OPER_LOWERLAYERDOWN, OPER_NOTPRESENT, - OPER_TESTING, OPER_UNKNOWN, RTATTR_START_OFFSET, RTM_NEWLINK, RTM_SETLINK, - RTM_GETLINK, MAX_SIZE) + OPER_TESTING, OPER_UNKNOWN, RTATTR_START_OFFSET, RTM_NEWLINK, RTM_DELLINK, + RTM_SETLINK, RTM_GETLINK, MAX_SIZE) def int_to_bytes(i): @@ -133,6 +134,75 @@ class TestParseNetlinkMessage(CiTestCase): str(context.exception)) +@mock.patch('cloudinit.sources.helpers.netlink.socket.socket') +@mock.patch('cloudinit.sources.helpers.netlink.read_netlink_socket') +class TestNicAttachDetach(CiTestCase): + with_logs = True + + def _media_switch_data(self, ifname, msg_type, operstate): + '''construct netlink data with specified fields''' + if ifname and operstate is not None: + data = bytearray(48) + bytes = ifname.encode("utf-8") + struct.pack_into("HH4sHHc", data, RTATTR_START_OFFSET, 8, 3, + bytes, 5, 16, int_to_bytes(operstate)) + elif ifname: + data = bytearray(40) + bytes = ifname.encode("utf-8") + struct.pack_into("HH4s", data, RTATTR_START_OFFSET, 8, 3, bytes) + elif operstate: + data = bytearray(40) + struct.pack_into("HHc", data, RTATTR_START_OFFSET, 5, 16, + int_to_bytes(operstate)) + struct.pack_into("=LHHLL", data, 0, len(data), msg_type, 0, 0, 0) + return data + + def test_nic_attached_oper_down(self, m_read_netlink_socket, m_socket): + '''Test for a new nic attached''' + ifname = "eth0" + data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) + m_read_netlink_socket.side_effect = [data_op_down] + ifread = wait_for_nic_attach_event(m_socket, []) + self.assertEqual(m_read_netlink_socket.call_count, 1) + self.assertEqual(ifname, ifread) + + def test_nic_attached_oper_up(self, m_read_netlink_socket, m_socket): + '''Test for a new nic attached''' + ifname = "eth0" + data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) + m_read_netlink_socket.side_effect = [data_op_up] + ifread = wait_for_nic_attach_event(m_socket, []) + self.assertEqual(m_read_netlink_socket.call_count, 1) + self.assertEqual(ifname, ifread) + + def test_nic_attach_ignore_existing(self, m_read_netlink_socket, m_socket): + '''Test that we read only the interfaces we are interested in.''' + data_eth0 = self._media_switch_data("eth0", RTM_NEWLINK, OPER_DOWN) + data_eth1 = self._media_switch_data("eth1", RTM_NEWLINK, OPER_DOWN) + m_read_netlink_socket.side_effect = [data_eth0, data_eth1] + ifread = wait_for_nic_attach_event(m_socket, ["eth0"]) + self.assertEqual(m_read_netlink_socket.call_count, 2) + self.assertEqual("eth1", ifread) + + def test_nic_attach_read_first(self, m_read_netlink_socket, m_socket): + '''Test that we read only the interfaces we are interested in.''' + data_eth0 = self._media_switch_data("eth0", RTM_NEWLINK, OPER_DOWN) + data_eth1 = self._media_switch_data("eth1", RTM_NEWLINK, OPER_DOWN) + m_read_netlink_socket.side_effect = [data_eth0, data_eth1] + ifread = wait_for_nic_attach_event(m_socket, ["eth1"]) + self.assertEqual(m_read_netlink_socket.call_count, 1) + self.assertEqual("eth0", ifread) + + def test_nic_detached(self, m_read_netlink_socket, m_socket): + '''Test for an existing nic detached''' + ifname = "eth0" + data_op_down = self._media_switch_data(ifname, RTM_DELLINK, OPER_DOWN) + m_read_netlink_socket.side_effect = [data_op_down] + ifread = wait_for_nic_detach_event(m_socket) + self.assertEqual(m_read_netlink_socket.call_count, 1) + self.assertEqual(ifname, ifread) + + @mock.patch('cloudinit.sources.helpers.netlink.socket.socket') @mock.patch('cloudinit.sources.helpers.netlink.read_netlink_socket') class TestWaitForMediaDisconnectConnect(CiTestCase): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 534314aa..e363c1f9 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -11,6 +11,7 @@ from cloudinit.version import version_string as vs from cloudinit.tests.helpers import ( HttprettyTestCase, CiTestCase, populate_dir, mock, wrap_and_call, ExitStack, resourceLocation) +from cloudinit.sources.helpers import netlink import copy import crypt @@ -78,6 +79,8 @@ def construct_valid_ovf_env(data=None, pubkeys=None, if platform_settings: for k, v in platform_settings.items(): content += "<%s>%s\n" % (k, v, k) + if "PreprovisionedVMType" not in platform_settings: + content += """""" content += """ """ @@ -156,6 +159,31 @@ SECONDARY_INTERFACE = { } } +IMDS_NETWORK_METADATA = { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "ipAddress": [] + }, + "ipv4": { + "subnet": [ + { + "prefix": "24", + "address": "10.0.0.0" + } + ], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81" + } + ] + } + } + ] +} + MOCKPATH = 'cloudinit.sources.DataSourceAzure.' @@ -366,8 +394,8 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01" @mock.patch(MOCKPATH + 'readurl') - @mock.patch(MOCKPATH + 'EphemeralDHCPv4') - @mock.patch(MOCKPATH + 'net.is_up') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4', autospec=True) + @mock.patch(MOCKPATH + 'net.is_up', autospec=True) def test_get_metadata_does_not_dhcp_if_network_is_up( self, m_net_is_up, m_dhcp, m_readurl): """Do not perform DHCP setup when nic is already up.""" @@ -384,9 +412,66 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): "Crawl of Azure Instance Metadata Service (IMDS) took", # log_time self.logs.getvalue()) - @mock.patch(MOCKPATH + 'readurl') - @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting') + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') + @mock.patch(MOCKPATH + 'net.is_up') + def test_get_compute_metadata_uses_compute_url( + self, m_net_is_up, m_dhcp, m_readurl): + """Make sure readurl is called with the correct url when accessing + network metadata""" + m_net_is_up.return_value = True + m_readurl.return_value = url_helper.StringResponse( + json.dumps(IMDS_NETWORK_METADATA).encode('utf-8')) + + dsaz.get_metadata_from_imds( + 'eth0', retries=3, md_type=dsaz.metadata_type.compute) + m_readurl.assert_called_with( + "http://169.254.169.254/metadata/instance?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, + timeout=mock.ANY) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') + @mock.patch(MOCKPATH + 'net.is_up') + def test_get_network_metadata_uses_network_url( + self, m_net_is_up, m_dhcp, m_readurl): + """Make sure readurl is called with the correct url when accessing + network metadata""" + m_net_is_up.return_value = True + m_readurl.return_value = url_helper.StringResponse( + json.dumps(IMDS_NETWORK_METADATA).encode('utf-8')) + + dsaz.get_metadata_from_imds( + 'eth0', retries=3, md_type=dsaz.metadata_type.network) + m_readurl.assert_called_with( + "http://169.254.169.254/metadata/instance/network?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, + timeout=mock.ANY) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') @mock.patch(MOCKPATH + 'net.is_up') + def test_get_default_metadata_uses_compute_url( + self, m_net_is_up, m_dhcp, m_readurl): + """Make sure readurl is called with the correct url when accessing + network metadata""" + m_net_is_up.return_value = True + m_readurl.return_value = url_helper.StringResponse( + json.dumps(IMDS_NETWORK_METADATA).encode('utf-8')) + + dsaz.get_metadata_from_imds( + 'eth0', retries=3) + m_readurl.assert_called_with( + "http://169.254.169.254/metadata/instance?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, + timeout=mock.ANY) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting', autospec=True) + @mock.patch(MOCKPATH + 'net.is_up', autospec=True) def test_get_metadata_performs_dhcp_when_network_is_down( self, m_net_is_up, m_dhcp, m_readurl): """Perform DHCP setup when nic is not up.""" @@ -410,7 +495,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS) @mock.patch('cloudinit.url_helper.time.sleep') - @mock.patch(MOCKPATH + 'net.is_up') + @mock.patch(MOCKPATH + 'net.is_up', autospec=True) def test_get_metadata_from_imds_empty_when_no_imds_present( self, m_net_is_up, m_sleep): """Return empty dict when IMDS network metadata is absent.""" @@ -431,7 +516,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): @mock.patch('requests.Session.request') @mock.patch('cloudinit.url_helper.time.sleep') - @mock.patch(MOCKPATH + 'net.is_up') + @mock.patch(MOCKPATH + 'net.is_up', autospec=True) def test_get_metadata_from_imds_retries_on_timeout( self, m_net_is_up, m_sleep, m_request): """Retry IMDS network metadata on timeout errors.""" @@ -801,6 +886,7 @@ scbus-1 on xpt0 bus 0 'sys_cfg': {}} dsrc = self._get_ds(data) expected_cfg = { + 'PreprovisionedVMType': None, 'PreprovisionedVm': False, 'datasource': {'Azure': {'agent_command': 'my_command'}}, 'system_info': {'default_user': {'name': u'myuser'}}} @@ -864,6 +950,66 @@ scbus-1 on xpt0 bus 0 dsrc.crawl_metadata() self.assertEqual(1, m_report_ready.call_count) + @mock.patch( + 'cloudinit.sources.DataSourceAzure.EphemeralDHCPv4WithReporting') + @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') + @mock.patch( + 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') + @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') + @mock.patch( + 'cloudinit.sources.DataSourceAzure.DataSourceAzure.' + '_wait_for_all_nics_ready') + def test_crawl_metadata_waits_for_nic_on_savable_vms( + self, detect_nics, poll_imds_func, report_ready_func, m_write, m_dhcp + ): + """If reprovisioning, report ready at the end""" + ovfenv = construct_valid_ovf_env( + platform_settings={"PreprovisionedVMType": "Savable", + "PreprovisionedVm": "True"} + ) + + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } + dsrc = self._get_ds(data) + poll_imds_func.return_value = ovfenv + dsrc.crawl_metadata() + self.assertEqual(1, report_ready_func.call_count) + self.assertEqual(1, detect_nics.call_count) + + @mock.patch( + 'cloudinit.sources.DataSourceAzure.EphemeralDHCPv4WithReporting') + @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') + @mock.patch( + 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') + @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') + @mock.patch( + 'cloudinit.sources.DataSourceAzure.DataSourceAzure.' + '_wait_for_all_nics_ready') + @mock.patch('os.path.isfile') + def test_detect_nics_when_marker_present( + self, is_file, detect_nics, poll_imds_func, report_ready_func, m_write, + m_dhcp): + """If reprovisioning, wait for nic attach if marker present""" + + def is_file_ret(key): + return key == dsaz.REPROVISION_NIC_ATTACH_MARKER_FILE + + is_file.side_effect = is_file_ret + ovfenv = construct_valid_ovf_env() + + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } + + dsrc = self._get_ds(data) + poll_imds_func.return_value = ovfenv + dsrc.crawl_metadata() + self.assertEqual(1, report_ready_func.call_count) + self.assertEqual(1, detect_nics.call_count) + @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') @mock.patch('cloudinit.sources.helpers.netlink.' 'wait_for_media_disconnect_connect') @@ -1526,7 +1672,7 @@ scbus-1 on xpt0 bus 0 @mock.patch('cloudinit.net.get_interface_mac') @mock.patch('cloudinit.net.get_devicelist') @mock.patch('cloudinit.net.device_driver') - @mock.patch('cloudinit.net.generate_fallback_config') + @mock.patch('cloudinit.net.generate_fallback_config', autospec=True) def test_fallback_network_config(self, mock_fallback, mock_dd, mock_devlist, mock_get_mac): """On absent IMDS network data, generate network fallback config.""" @@ -1561,7 +1707,7 @@ scbus-1 on xpt0 bus 0 blacklist_drivers=['mlx4_core', 'mlx5_core'], config_driver=True) - @mock.patch(MOCKPATH + 'net.get_interfaces') + @mock.patch(MOCKPATH + 'net.get_interfaces', autospec=True) @mock.patch(MOCKPATH + 'util.is_FreeBSD') def test_blacklist_through_distro( self, m_is_freebsd, m_net_get_interfaces): @@ -1583,17 +1729,17 @@ scbus-1 on xpt0 bus 0 m_net_get_interfaces.assert_called_with( blacklist_drivers=dsaz.BLACKLIST_DRIVERS) - @mock.patch(MOCKPATH + 'subp.subp') + @mock.patch(MOCKPATH + 'subp.subp', autospec=True) def test_get_hostname_with_no_args(self, m_subp): dsaz.get_hostname() m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'subp.subp') + @mock.patch(MOCKPATH + 'subp.subp', autospec=True) def test_get_hostname_with_string_arg(self, m_subp): dsaz.get_hostname(hostname_command="hostname") m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'subp.subp') + @mock.patch(MOCKPATH + 'subp.subp', autospec=True) def test_get_hostname_with_iterable_arg(self, m_subp): dsaz.get_hostname(hostname_command=("hostname",)) m_subp.assert_called_once_with(("hostname",), capture=True) @@ -2224,6 +2370,29 @@ class TestPreprovisioningReadAzureOvfFlag(CiTestCase): ret = dsaz.read_azure_ovf(content) cfg = ret[2] self.assertFalse(cfg['PreprovisionedVm']) + self.assertEqual(None, cfg["PreprovisionedVMType"]) + + def test_read_azure_ovf_with_running_type(self): + """The read_azure_ovf method should set PreprovisionedVMType + cfg flag to Running.""" + content = construct_valid_ovf_env( + platform_settings={"PreprovisionedVMType": "Running", + "PreprovisionedVm": "True"}) + ret = dsaz.read_azure_ovf(content) + cfg = ret[2] + self.assertTrue(cfg['PreprovisionedVm']) + self.assertEqual("Running", cfg['PreprovisionedVMType']) + + def test_read_azure_ovf_with_savable_type(self): + """The read_azure_ovf method should set PreprovisionedVMType + cfg flag to Savable.""" + content = construct_valid_ovf_env( + platform_settings={"PreprovisionedVMType": "Savable", + "PreprovisionedVm": "True"}) + ret = dsaz.read_azure_ovf(content) + cfg = ret[2] + self.assertTrue(cfg['PreprovisionedVm']) + self.assertEqual("Savable", cfg['PreprovisionedVMType']) @mock.patch('os.path.isfile') @@ -2273,6 +2442,227 @@ class TestPreprovisioningShouldReprovision(CiTestCase): _poll_imds.assert_called_with() +class TestPreprovisioningHotAttachNics(CiTestCase): + + def setUp(self): + super(TestPreprovisioningHotAttachNics, self).setUp() + self.tmp = self.tmp_dir() + self.waagent_d = self.tmp_path('/var/lib/waagent', self.tmp) + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_detach_event', + autospec=True) + @mock.patch(MOCKPATH + 'util.write_file', autospec=True) + def test_nic_detach_writes_marker(self, m_writefile, m_detach): + """When we detect that a nic gets detached, we write a marker for it""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + nl_sock = mock.MagicMock() + dsa._wait_for_nic_detach(nl_sock) + m_detach.assert_called_with(nl_sock) + self.assertEqual(1, m_detach.call_count) + m_writefile.assert_called_with( + dsaz.REPROVISION_NIC_DETACHED_MARKER_FILE, mock.ANY) + + @mock.patch(MOCKPATH + 'util.write_file', autospec=True) + @mock.patch(MOCKPATH + 'DataSourceAzure.fallback_interface') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting') + @mock.patch(MOCKPATH + 'DataSourceAzure._report_ready') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + def test_detect_nic_attach_reports_ready_and_waits_for_detach( + self, m_detach, m_report_ready, m_dhcp, m_fallback_if, + m_writefile): + """Report ready first and then wait for nic detach""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa._wait_for_all_nics_ready() + m_fallback_if.return_value = "Dummy interface" + self.assertEqual(1, m_report_ready.call_count) + self.assertEqual(1, m_detach.call_count) + self.assertEqual(1, m_writefile.call_count) + self.assertEqual(1, m_dhcp.call_count) + m_writefile.assert_called_with(dsaz.REPORTED_READY_MARKER_FILE, + mock.ANY) + + @mock.patch('os.path.isfile') + @mock.patch(MOCKPATH + 'DataSourceAzure.fallback_interface') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting') + @mock.patch(MOCKPATH + 'DataSourceAzure._report_ready') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + def test_detect_nic_attach_skips_report_ready_when_marker_present( + self, m_detach, m_report_ready, m_dhcp, m_fallback_if, m_isfile): + """Skip reporting ready if we already have a marker file.""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + + def isfile(key): + return key == dsaz.REPORTED_READY_MARKER_FILE + + m_isfile.side_effect = isfile + dsa._wait_for_all_nics_ready() + m_fallback_if.return_value = "Dummy interface" + self.assertEqual(0, m_report_ready.call_count) + self.assertEqual(0, m_dhcp.call_count) + self.assertEqual(1, m_detach.call_count) + + @mock.patch('os.path.isfile') + @mock.patch(MOCKPATH + 'DataSourceAzure.fallback_interface') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting') + @mock.patch(MOCKPATH + 'DataSourceAzure._report_ready') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + def test_detect_nic_attach_skips_nic_detach_when_marker_present( + self, m_detach, m_report_ready, m_dhcp, m_fallback_if, m_isfile): + """Skip wait for nic detach if it already happened.""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + + m_isfile.return_value = True + dsa._wait_for_all_nics_ready() + m_fallback_if.return_value = "Dummy interface" + self.assertEqual(0, m_report_ready.call_count) + self.assertEqual(0, m_dhcp.call_count) + self.assertEqual(0, m_detach.call_count) + + @mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up', autospec=True) + @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event') + @mock.patch('cloudinit.sources.net.find_fallback_nic') + @mock.patch(MOCKPATH + 'get_metadata_from_imds') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + @mock.patch('os.path.isfile') + def test_wait_for_nic_attach_if_no_fallback_interface( + self, m_isfile, m_detach, m_dhcpv4, m_imds, m_fallback_if, + m_attach, m_link_up): + """Wait for nic attach if we do not have a fallback interface""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + lease = { + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'} + + m_isfile.return_value = True + m_attach.return_value = "eth0" + dhcp_ctx = mock.MagicMock(lease=lease) + dhcp_ctx.obtain_lease.return_value = lease + m_dhcpv4.return_value = dhcp_ctx + m_imds.return_value = IMDS_NETWORK_METADATA + m_fallback_if.return_value = None + + dsa._wait_for_all_nics_ready() + + self.assertEqual(0, m_detach.call_count) + self.assertEqual(1, m_attach.call_count) + self.assertEqual(1, m_dhcpv4.call_count) + self.assertEqual(1, m_imds.call_count) + self.assertEqual(1, m_link_up.call_count) + m_link_up.assert_called_with(mock.ANY, "eth0") + + @mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up') + @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event') + @mock.patch('cloudinit.sources.net.find_fallback_nic') + @mock.patch(MOCKPATH + 'get_metadata_from_imds') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + @mock.patch('os.path.isfile') + def test_wait_for_nic_attach_multinic_attach( + self, m_isfile, m_detach, m_dhcpv4, m_imds, m_fallback_if, + m_attach, m_link_up): + """Wait for nic attach if we do not have a fallback interface""" + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + lease = { + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'} + m_attach_call_count = 0 + + def nic_attach_ret(nl_sock, nics_found): + nonlocal m_attach_call_count + if m_attach_call_count == 0: + m_attach_call_count = m_attach_call_count + 1 + return "eth0" + return "eth1" + + def network_metadata_ret(ifname, retries, type): + # Simulate two NICs by adding the same one twice. + md = IMDS_NETWORK_METADATA + md['interface'].append(md['interface'][0]) + if ifname == "eth0": + return md + raise requests.Timeout('Fake connection timeout') + + m_isfile.return_value = True + m_attach.side_effect = nic_attach_ret + dhcp_ctx = mock.MagicMock(lease=lease) + dhcp_ctx.obtain_lease.return_value = lease + m_dhcpv4.return_value = dhcp_ctx + m_imds.side_effect = network_metadata_ret + m_fallback_if.return_value = None + + dsa._wait_for_all_nics_ready() + + self.assertEqual(0, m_detach.call_count) + self.assertEqual(2, m_attach.call_count) + # DHCP and network metadata calls will only happen on the primary NIC. + self.assertEqual(1, m_dhcpv4.call_count) + self.assertEqual(1, m_imds.call_count) + self.assertEqual(2, m_link_up.call_count) + + @mock.patch('cloudinit.distros.networking.LinuxNetworking.try_set_link_up') + def test_wait_for_link_up_returns_if_already_up( + self, m_is_link_up): + """Waiting for link to be up should return immediately if the link is + already up.""" + + distro_cls = distros.fetch('ubuntu') + distro = distro_cls('ubuntu', {}, self.paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + m_is_link_up.return_value = True + + dsa.wait_for_link_up("eth0") + self.assertEqual(1, m_is_link_up.call_count) + + @mock.patch(MOCKPATH + 'util.write_file') + @mock.patch('cloudinit.net.read_sys_net') + @mock.patch('cloudinit.distros.networking.LinuxNetworking.try_set_link_up') + def test_wait_for_link_up_writes_to_device_file( + self, m_is_link_up, m_read_sys_net, m_writefile): + """Waiting for link to be up should return immediately if the link is + already up.""" + + distro_cls = distros.fetch('ubuntu') + distro = distro_cls('ubuntu', {}, self.paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + + callcount = 0 + + def linkup(key): + nonlocal callcount + if callcount == 0: + callcount += 1 + return False + return True + + m_is_link_up.side_effect = linkup + + dsa.wait_for_link_up("eth0") + self.assertEqual(2, m_is_link_up.call_count) + self.assertEqual(1, m_read_sys_net.call_count) + self.assertEqual(2, m_writefile.call_count) + + @mock.patch('cloudinit.sources.helpers.netlink.' + 'create_bound_netlink_socket') + def test_wait_for_all_nics_ready_raises_if_socket_fails(self, m_socket): + """Waiting for all nics should raise exception if netlink socket + creation fails.""" + + m_socket.side_effect = netlink.NetlinkCreateSocketError + distro_cls = distros.fetch('ubuntu') + distro = distro_cls('ubuntu', {}, self.paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + + self.assertRaises(netlink.NetlinkCreateSocketError, + dsa._wait_for_all_nics_ready) + # dsa._wait_for_all_nics_ready() + + @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('cloudinit.sources.helpers.netlink.' @@ -2330,6 +2720,24 @@ class TestPreprovisioningPollIMDS(CiTestCase): self.assertEqual(3, m_dhcpv4.call_count, 'Expected 3 DHCP calls') self.assertEqual(4, self.tries, 'Expected 4 total reads from IMDS') + @mock.patch('os.path.isfile') + def test_poll_imds_skips_dhcp_if_ctx_present( + self, m_isfile, report_ready_func, fake_resp, m_media_switch, + m_dhcp, m_net): + """The poll_imds function should reuse the dhcp ctx if it is already + present. This happens when we wait for nic to be hot-attached before + polling for reprovisiondata. Note that if this ctx is set when + _poll_imds is called, then it is not expected to be waiting for + media_disconnect_connect either.""" + report_file = self.tmp_path('report_marker', self.tmp) + m_isfile.return_value = True + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa._ephemeral_dhcp_ctx = "Dummy dhcp ctx" + with mock.patch(MOCKPATH + 'REPORTED_READY_MARKER_FILE', report_file): + dsa._poll_imds() + self.assertEqual(0, m_dhcp.call_count) + self.assertEqual(0, m_media_switch.call_count) + def test_does_not_poll_imds_report_ready_when_marker_file_exists( self, m_report_ready, m_request, m_media_switch, m_dhcp, m_net): """poll_imds should not call report ready when the reported ready @@ -2390,7 +2798,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): @mock.patch(MOCKPATH + 'util.is_FreeBSD') @mock.patch('cloudinit.sources.helpers.netlink.' 'wait_for_media_disconnect_connect') -@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') +@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network', autospec=True) @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('requests.Session.request') class TestAzureDataSourcePreprovisioning(CiTestCase): @@ -2412,7 +2820,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): m_dhcp.return_value = [{ 'interface': 'eth9', 'fixed-address': '192.168.2.9', 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0'}] - url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02' + url = 'http://{0}/metadata/reprovisiondata?api-version=2019-06-01' host = "169.254.169.254" full_url = url.format(host) m_request.return_value = mock.MagicMock(status_code=200, text="ovf", @@ -2445,7 +2853,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): 'interface': 'eth9', 'fixed-address': '192.168.2.9', 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', 'unknown-245': '624c3620'}] - url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02' + url = 'http://{0}/metadata/reprovisiondata?api-version=2019-06-01' host = "169.254.169.254" full_url = url.format(host) hostname = "myhost" diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 9b594a44..1e0c3ea4 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,6 +1,7 @@ ader1990 AlexBaranowski Aman306 +aswinrajamannar beezly bipinbachhao BirknerAlex -- cgit v1.2.3 From 66a851aca7acf3ae39cb5c03fc97c563716b5aa3 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 23 Nov 2020 12:20:29 -0500 Subject: .travis.yml: correctly integration test the built .deb (#683) `IMAGE_SOURCE` was renamed to `CLOUD_INIT_SOURCE` before the testing framework ever landed, but I did not mirror that change in .travis.yml before it landed. This addresses that. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6d25f477..2fad49f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -199,7 +199,7 @@ matrix: fi # Use sudo to get a new shell where we're in the sbuild group - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' - - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' & + - sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' & - | SECONDS=0 while [ -e /proc/$! ]; do -- cgit v1.2.3 From e454dea5855019a5acdd6acafdef2ae07d069235 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 23 Nov 2020 11:50:57 -0600 Subject: Integration test for fallocate falling back to dd (#681) See #585 --- tests/integration_tests/bugs/test_lp1897099.py | 31 ++++++++++++++++++++++++++ tests/integration_tests/conftest.py | 15 ++++++++++--- tox.ini | 1 + 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/integration_tests/bugs/test_lp1897099.py diff --git a/tests/integration_tests/bugs/test_lp1897099.py b/tests/integration_tests/bugs/test_lp1897099.py new file mode 100644 index 00000000..27c8927f --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1897099.py @@ -0,0 +1,31 @@ +""" Integration test for LP #187099 + +Ensure that if fallocate fails during mkswap that we fall back to using dd + +https://bugs.launchpad.net/cloud-init/+bug/1897099 +""" + +import pytest + + +USER_DATA = """\ +#cloud-config +bootcmd: + - echo 'whoops' > /usr/bin/fallocate +swap: + filename: /swap.img + size: 10000000 + maxsize: 10000000 +""" + + +@pytest.mark.sru_2020_11 +@pytest.mark.user_data(USER_DATA) +@pytest.mark.no_container('Containers cannot configure swap') +def test_fallocate_fallback(client): + log = client.read_from_file('/var/log/cloud-init.log') + assert '/swap.img' in client.execute('cat /proc/swaps') + assert '/swap.img' in client.execute('cat /etc/fstab') + assert 'fallocate swap creation failed, will attempt with dd' in log + assert "Running command ['dd', 'if=/dev/zero', 'of=/swap.img'" in log + assert 'SUCCESS: config-mounts ran successfully' in log diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index e31a9192..54867096 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -37,11 +37,20 @@ def pytest_runtest_setup(item): specified, then we assume the test can be run anywhere. """ all_platforms = platforms.keys() - supported_platforms = set(all_platforms).intersection( - mark.name for mark in item.iter_markers()) + test_marks = [mark.name for mark in item.iter_markers()] + supported_platforms = set(all_platforms).intersection(test_marks) current_platform = integration_settings.PLATFORM + unsupported_message = 'Cannot run on platform {}'.format(current_platform) + if 'no_container' in test_marks: + if 'lxd_container' in test_marks: + raise Exception( + 'lxd_container and no_container marks simultaneously set ' + 'on test' + ) + if current_platform == 'lxd_container': + pytest.skip(unsupported_message) if supported_platforms and current_platform not in supported_platforms: - pytest.skip('Cannot run on platform {}'.format(current_platform)) + pytest.skip(unsupported_message) # disable_subp_usage is defined at a higher level, but we don't diff --git a/tox.ini b/tox.ini index 066f923a..f08e0f03 100644 --- a/tox.ini +++ b/tox.ini @@ -167,6 +167,7 @@ markers = azure: test will only run on Azure platform oci: test will only run on OCI platform lxd_container: test will only run in LXD container + no_container: test cannot run in a container user_data: the user data to be passed to the test instance instance_name: the name to be used for the test instance sru_2020_11: test is part of the 2020/11 SRU verification -- cgit v1.2.3 From 8a493bf08d8b09d4f3a35dae725756d157844201 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 23 Nov 2020 12:12:52 -0600 Subject: LXD VM support in integration tests (#678) --- integration-requirements.txt | 2 +- tests/integration_tests/clouds.py | 78 +++++++++++++++++++++++++----------- tests/integration_tests/conftest.py | 4 +- tests/integration_tests/instances.py | 2 +- tox.ini | 1 + 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/integration-requirements.txt b/integration-requirements.txt index 61d2e504..e8ddb648 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -pycloudlib @ git+https://github.com/canonical/pycloudlib.git@e6b2b3732a2a17d48bdf1167f56eb14576215d3c +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@9211c0e5b34794595565d4626bc41ddbe14994f2 pytest diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 2841261b..88ac4408 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -2,7 +2,8 @@ from abc import ABC, abstractmethod import logging -from pycloudlib import EC2, GCE, Azure, OCI, LXD +from pycloudlib import EC2, GCE, Azure, OCI, LXDContainer, LXDVirtualMachine +from pycloudlib.lxd.instance import LXDInstance import cloudinit from cloudinit.subp import subp @@ -12,7 +13,7 @@ from tests.integration_tests.instances import ( IntegrationGceInstance, IntegrationAzureInstance, IntegrationInstance, IntegrationOciInstance, - IntegrationLxdContainerInstance, + IntegrationLxdInstance, ) try: @@ -143,20 +144,48 @@ class OciCloud(IntegrationCloud): ) -class LxdContainerCloud(IntegrationCloud): - datasource = 'lxd_container' - integration_instance_cls = IntegrationLxdContainerInstance +class _LxdIntegrationCloud(IntegrationCloud): + integration_instance_cls = IntegrationLxdInstance def _get_cloud_instance(self): - return LXD(tag='lxd-integration-test') + return self.pycloudlib_instance_cls(tag=self.instance_tag) + + @staticmethod + def _get_or_set_profile_list(release): + return None + + @staticmethod + def _mount_source(instance: LXDInstance): + target_path = '/usr/lib/python3/dist-packages/cloudinit' + format_variables = { + 'name': instance.name, + 'source_path': cloudinit.__path__[0], + 'container_path': target_path, + } + log.info( + 'Mounting source {source_path} directly onto LXD container/vm ' + 'named {name} at {container_path}'.format(**format_variables)) + command = ( + 'lxc config device add {name} host-cloud-init disk ' + 'source={source_path} ' + 'path={container_path}' + ).format(**format_variables) + subp(command.split()) def _perform_launch(self, launch_kwargs): launch_kwargs['inst_type'] = launch_kwargs.pop('instance_type', None) launch_kwargs.pop('wait') + release = launch_kwargs.pop('image_id') + + try: + profile_list = launch_kwargs['profile_list'] + except KeyError: + profile_list = self._get_or_set_profile_list(release) pycloudlib_instance = self.cloud_instance.init( launch_kwargs.pop('name', None), - launch_kwargs.pop('image_id'), + release, + profile_list=profile_list, **launch_kwargs ) if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': @@ -165,19 +194,22 @@ class LxdContainerCloud(IntegrationCloud): pycloudlib_instance.wait(raise_on_cloudinit_failure=False) return pycloudlib_instance - def _mount_source(self, instance): - container_path = '/usr/lib/python3/dist-packages/cloudinit' - format_variables = { - 'name': instance.name, - 'cloudinit_path': cloudinit.__path__[0], - 'container_path': container_path, - } - log.info( - 'Mounting source {cloudinit_path} directly onto LXD container/vm ' - 'named {name} at {container_path}'.format(**format_variables)) - command = ( - 'lxc config device add {name} host-cloud-init disk ' - 'source={cloudinit_path} ' - 'path={container_path}' - ).format(**format_variables) - subp(command.split()) + +class LxdContainerCloud(_LxdIntegrationCloud): + datasource = 'lxd_container' + pycloudlib_instance_cls = LXDContainer + instance_tag = 'lxd-container-integration-test' + + +class LxdVmCloud(_LxdIntegrationCloud): + datasource = 'lxd_vm' + pycloudlib_instance_cls = LXDVirtualMachine + instance_tag = 'lxd-vm-integration-test' + _profile_list = None + + def _get_or_set_profile_list(self, release): + if self._profile_list: + return self._profile_list + self._profile_list = self.cloud_instance.build_necessary_profiles( + release) + return self._profile_list diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 54867096..73b44bfc 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -12,6 +12,7 @@ from tests.integration_tests.clouds import ( AzureCloud, OciCloud, LxdContainerCloud, + LxdVmCloud, ) @@ -25,6 +26,7 @@ platforms = { 'azure': AzureCloud, 'oci': OciCloud, 'lxd_container': LxdContainerCloud, + 'lxd_vm': LxdVmCloud, } @@ -87,7 +89,7 @@ def setup_image(session_cloud): if integration_settings.CLOUD_INIT_SOURCE == 'NONE': pass # that was easy elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': - if session_cloud.datasource != 'lxd_container': + if session_cloud.datasource not in ['lxd_container', 'lxd_vm']: raise ValueError( 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') # The mount needs to happen after the instance is created, so diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 67a6fb92..ca0b38d5 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -125,5 +125,5 @@ class IntegrationOciInstance(IntegrationInstance): pass -class IntegrationLxdContainerInstance(IntegrationInstance): +class IntegrationLxdInstance(IntegrationInstance): use_sudo = False diff --git a/tox.ini b/tox.ini index f08e0f03..30b11398 100644 --- a/tox.ini +++ b/tox.ini @@ -167,6 +167,7 @@ markers = azure: test will only run on Azure platform oci: test will only run on OCI platform lxd_container: test will only run in LXD container + lxd_vm: test will only run in LXD VM no_container: test cannot run in a container user_data: the user data to be passed to the test instance instance_name: the name to be used for the test instance -- cgit v1.2.3 From 6e86d2a5649b3a9113923c73154ebf02224732a6 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 23 Nov 2020 15:52:19 -0600 Subject: Ensure proper root permissions in integration tests (#664) Tests previously assumed that when executing commands and transferring files that user will have root permissions. This change updated integration testing infrastructure so that is true. --- integration-requirements.txt | 2 +- tests/integration_tests/instances.py | 45 +++++++++++++++++----- .../integration_tests/modules/test_users_groups.py | 5 ++- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/integration-requirements.txt b/integration-requirements.txt index e8ddb648..3648a0f1 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -pycloudlib @ git+https://github.com/canonical/pycloudlib.git@9211c0e5b34794595565d4626bc41ddbe14994f2 +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@4b8d2cd5ac6316810ce16d081842da575625ca4f pytest diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index ca0b38d5..9b13288c 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -1,9 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging import os +import uuid from tempfile import NamedTemporaryFile from pycloudlib.instance import BaseInstance +from pycloudlib.result import Result from tests.integration_tests import integration_settings @@ -18,6 +20,11 @@ except ImportError: log = logging.getLogger('integration_testing') +def _get_tmp_path(): + tmp_filename = str(uuid.uuid4()) + return '/var/tmp/{}.tmp'.format(tmp_filename) + + class IntegrationInstance: use_sudo = True @@ -30,21 +37,39 @@ class IntegrationInstance: def destroy(self): self.instance.delete() - def execute(self, command): - return self.instance.execute(command) + def execute(self, command, *, use_sudo=None) -> Result: + if self.instance.username == 'root' and use_sudo is False: + raise Exception('Root user cannot run unprivileged') + if use_sudo is None: + use_sudo = self.use_sudo + return self.instance.execute(command, use_sudo=use_sudo) - def pull_file(self, remote_file, local_file): - self.instance.pull_file(remote_file, local_file) + def pull_file(self, remote_path, local_path): + # First copy to a temporary directory because of permissions issues + tmp_path = _get_tmp_path() + self.instance.execute('cp {} {}'.format(remote_path, tmp_path)) + self.instance.pull_file(tmp_path, local_path) def push_file(self, local_path, remote_path): - self.instance.push_file(local_path, remote_path) + # First push to a temporary directory because of permissions issues + tmp_path = _get_tmp_path() + self.instance.push_file(local_path, tmp_path) + self.execute('mv {} {}'.format(tmp_path, remote_path)) def read_from_file(self, remote_path) -> str: - tmp_file = NamedTemporaryFile('r') - self.pull_file(remote_path, tmp_file.name) - with tmp_file as f: - contents = f.read() - return contents + result = self.execute('cat {}'.format(remote_path)) + if result.failed: + # TODO: Raise here whatever pycloudlib raises when it has + # a consistent error response + raise IOError( + 'Failed reading remote file via cat: {}\n' + 'Return code: {}\n' + 'Stderr: {}\n' + 'Stdout: {}'.format( + remote_path, result.return_code, + result.stderr, result.stdout) + ) + return result.stdout def write_to_file(self, remote_path, contents: str): # Writes file locally and then pushes it rather diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 6a085a8f..6a51f5a6 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -70,7 +70,10 @@ class TestUsersGroups: def test_users_groups(self, regex, getent_args, class_client): """Use getent to interrogate the various expected outcomes""" result = class_client.execute(["getent"] + getent_args) - assert re.search(regex, result.stdout) is not None + assert re.search(regex, result.stdout) is not None, ( + "'getent {}' resulted in '{}', " + "but expected to match regex {}".format( + ' '.join(getent_args), result.stdout, regex)) def test_user_root_in_secret(self, class_client): """Test root user is in 'secret' group.""" -- cgit v1.2.3 From 87df9d6cac95112560e5c652a68b74a28f4a6785 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 24 Nov 2020 07:39:56 -0700 Subject: tox: avoid tox testenv subsvars for xenial support (#684) Xenial tox 2.3.1 still suffers from https://github.com/tox-dev/tox/issues/208. As a result we cannot use testenv substvars if we want to support running tox on xenial systems. Drop the {[testenv:integration-tests]*} substitutions to avoid tracebacks when running `tox -e xenial-dev` which has the signature: "No support for the %s substitution type" % sub_type) tox.ConfigError: ConfigError: No support for the posargs substitution type --- tox.ini | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 30b11398..022b918d 100644 --- a/tox.ini +++ b/tox.ini @@ -139,6 +139,12 @@ passenv = HOME TRAVIS deps = -r{toxinidir}/cloud-tests-requirements.txt +# Until Xenial tox support is dropped or bumps to tox:2.3.2, reflect changes to +# deps into testenv:integration-tests-ci: commands, passenv and deps. +# This is due to (https://github.com/tox-dev/tox/issues/208) which means that +# the {posargs} handling and substitutions won't do what we want until tox 2.3.2 +# Once Xenial is dropped, integration-tests-ci can use proper substitution +# commands = {[testenv:integration-tests]commands} [testenv:integration-tests] basepython = python3 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} @@ -147,9 +153,10 @@ deps = -r{toxinidir}/integration-requirements.txt [testenv:integration-tests-ci] -commands = {[testenv:integration-tests]commands} -passenv = {[testenv:integration-tests]passenv} -deps = {[testenv:integration-tests]deps} +commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} +passenv = CLOUD_INIT_* +deps = + -r{toxinidir}/integration-requirements.txt setenv = PYTEST_ADDOPTS="-m ci" -- cgit v1.2.3 From 47f4229ebcef9f83df8b549bb869a2dbf6dff17c Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 24 Nov 2020 11:38:50 -0600 Subject: Release 20.4 (#686) Bump the version in cloudinit/version.py to 20.4 and update ChangeLog. LP: #1905440 --- ChangeLog | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++ cloudinit/version.py | 2 +- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3e680736..33b2bf74 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,149 @@ +20.4 + - tox: avoid tox testenv subsvars for xenial support (#684) + - Ensure proper root permissions in integration tests (#664) [James Falcon] + - LXD VM support in integration tests (#678) [James Falcon] + - Integration test for fallocate falling back to dd (#681) [James Falcon] + - .travis.yml: correctly integration test the built .deb (#683) + - Ability to hot-attach NICs to preprovisioned VMs before reprovisioning + (#613) [aswinrajamannar] + - Support configuring SSH host certificates. (#660) [Jonathan Lung] + - add integration test for LP: #1900837 (#679) + - cc_resizefs on FreeBSD: Fix _can_skip_ufs_resize (#655) + [Mina Galić] (LP: #1901958, #1901958) + - DataSourceAzure: push dmesg log to KVP (#670) [Anh Vo] + - Make mount in place for tests work (#667) [James Falcon] + - integration_tests: restore emission of settings to log (#657) + - DataSourceAzure: update password for defuser if exists (#671) [Anh Vo] + - tox.ini: only select "ci" marked tests for CI runs (#677) + - Azure helper: Increase Azure Endpoint HTTP retries (#619) [Johnson Shi] + - DataSourceAzure: send failure signal on Azure datasource failure (#594) + [Johnson Shi] + - test_persistence: simplify VersionIsPoppedFromState (#674) + - only run a subset of integration tests in CI (#672) + - cli: add --system param to allow validating system user-data on a + machine (#575) + - test_persistence: add VersionIsPoppedFromState test (#673) + - introduce an upgrade framework and related testing (#659) + - add --no-tty option to gpg (#669) [Till Riedel] (LP: #1813396) + - Pin pycloudlib to a working commit (#666) [James Falcon] + - DataSourceOpenNebula: exclude SRANDOM from context output (#665) + - cloud_tests: add hirsute release definition (#662) + - split integration and cloud_tests requirements (#652) + - faq.rst: add warning to answer that suggests running `clean` (#661) + - Fix stacktrace in DataSourceRbxCloud if no metadata disk is found (#632) + [Scott Moser] + - Make wakeonlan Network Config v2 setting actually work (#626) + [dermotbradley] + - HACKING.md: unify network-refactoring namespace (#658) [Mina Galić] + - replace usage of dmidecode with kenv on FreeBSD (#621) [Mina Galić] + - Prevent timeout on travis integration tests. (#651) [James Falcon] + - azure: enable pushing the log to KVP from the last pushed byte (#614) + [Moustafa Moustafa] + - Fix launch_kwargs bug in integration tests (#654) [James Falcon] + - split read_fs_info into linux & freebsd parts (#625) [Mina Galić] + - PULL_REQUEST_TEMPLATE.md: expand commit message section (#642) + - Make some language improvements in growpart documentation (#649) + [Shane Frasier] + - Revert ".travis.yml: use a known-working version of lxd (#643)" (#650) + - Fix not sourcing default 50-cloud-init ENI file on Debian (#598) + [WebSpider] + - remove unnecessary reboot from gpart resize (#646) [Mina Galić] + - cloudinit: move dmi functions out of util (#622) [Scott Moser] + - integration_tests: various launch improvements (#638) + - test_lp1886531: don't assume /etc/fstab exists (#639) + - Remove Ubuntu restriction from PR template (#648) [James Falcon] + - util: fix mounting of vfat on *BSD (#637) [Mina Galić] + - conftest: improve docstring for disable_subp_usage (#644) + - doc: add example query commands to debug Jinja templates (#645) + - Correct documentation and testcase data for some user-data YAML (#618) + [dermotbradley] + - Hetzner: Fix instance_id / SMBIOS serial comparison (#640) + [Markus Schade] + - .travis.yml: use a known-working version of lxd (#643) + - tools/build-on-freebsd: fix comment explaining purpose of the script + (#635) [Mina Galić] + - Hetzner: initialize instance_id from system-serial-number (#630) + [Markus Schade] (LP: #1885527) + - Explicit set IPV6_AUTOCONF and IPV6_FORCE_ACCEPT_RA on static6 (#634) + [Eduardo Otubo] + - get_interfaces: don't exclude Open vSwitch bridge/bond members (#608) + [Lukas Märdian] (LP: #1898997) + - Add config modules for controlling IBM PowerVM RMC. (#584) + [Aman306] (LP: #1895979) + - Update network config docs to clarify MAC address quoting (#623) + [dermotbradley] + - gentoo: fix hostname rendering when value has a comment (#611) + [Manuel Aguilera] + - refactor integration testing infrastructure (#610) [James Falcon] + - stages: don't reset permissions of cloud-init.log every boot (#624) + (LP: #1900837) + - docs: Add how to use cloud-localds to boot qemu (#617) [Joshua Powers] + - Drop vestigial update_resolve_conf_file function (#620) [Scott Moser] + - cc_mounts: correctly fallback to dd if fallocate fails (#585) + (LP: #1897099) + - .travis.yml: add integration-tests to Travis matrix (#600) + - ssh_util: handle non-default AuthorizedKeysFile config (#586) + [Eduardo Otubo] + - Multiple file fix for AuthorizedKeysFile config (#60) [Eduardo Otubo] + - bddeb: new --packaging-branch argument to pull packaging from branch + (#576) [Paride Legovini] + - Add more integration tests (#615) [lucasmoura] + - DataSourceAzure: write marker file after report ready in preprovisioning + (#590) [Johnson Shi] + - integration_tests: emit settings to log during setup (#601) + - integration_tests: implement citest tests run in Travis (#605) + - Add Azure support to integration test framework (#604) [James Falcon] + - openstack: consider product_name as valid chassis tag (#580) + [Adrian Vladu] (LP: #1895976) + - azure: clean up and refactor report_diagnostic_event (#563) [Johnson Shi] + - net: add the ability to blacklist network interfaces based on driver + during enumeration of physical network devices (#591) [Anh Vo] + - integration_tests: don't error on cloud-init failure (#596) + - integration_tests: improve cloud-init.log assertions (#593) + - conftest.py: remove top-level import of httpretty (#599) + - tox.ini: add integration-tests testenv definition (#595) + - PULL_REQUEST_TEMPLATE.md: empty checkboxes need a space (#597) + - add integration test for LP: #1886531 (#592) + - Initial implementation of integration testing infrastructure (#581) + [James Falcon] + - Fix name of ntp and chrony service on CentOS and RHEL. (#589) + [Scott Moser] (LP: #1897915) + - Adding a PR template (#587) [James Falcon] + - Azure parse_network_config uses fallback cfg when generate IMDS network + cfg fails (#549) [Johnson Shi] + - features: refresh docs for easier out-of-context reading (#582) + - Fix typo in resolv_conf module's description (#578) [Wacław Schiller] + - cc_users_groups: minor doc formatting fix (#577) + - Fix typo in disk_setup module's description (#579) [Wacław Schiller] + - Add vendor-data support to seedfrom parameter for NoCloud and OVF (#570) + [Johann Queuniet] + - boot.rst: add First Boot Determination section (#568) (LP: #1888858) + - opennebula.rst: minor readability improvements (#573) [Mina Galić] + - cloudinit: remove unused LOG variables (#574) + - create a shutdown_command method in distro classes (#567) + [Emmanuel Thomé] + - user_data: remove unused constant (#566) + - network: Fix type and respect name when rendering vlan in + sysconfig. (#541) [Eduardo Otubo] (LP: #1788915, #1826608) + - Retrieve SSH keys from IMDS first with OVF as a fallback (#509) + [Thomas Stringer] + - Add jqueuniet as contributor (#569) [Johann Queuniet] + - distros: minor typo fix (#562) + - Bump the integration-requirements versioned dependencies (#565) + [Paride Legovini] + - network-config-format-v1: fix typo in nameserver example (#564) + [Stanislas] + - Run cloud-init-local.service after the hv_kvp_daemon (#505) + [Robert Schweikert] + - Add method type hints for Azure helper (#540) [Johnson Shi] + - systemd: add Before=shutdown.target when Conflicts=shutdown.target is + used (#546) [Paride Legovini] + - LXD: detach network from profile before deleting it (#542) + [Paride Legovini] (LP: #1776958) + - redhat spec: add missing BuildRequires (#552) [Paride Legovini] + - util: remove debug statement (#556) [Joshua Powers] + - Fix cloud config on chef example (#551) [lucasmoura] + 20.3 - Azure: Add netplan driver filter when using hv_netvsc driver (#539) [James Falcon] (LP: #1830740) diff --git a/cloudinit/version.py b/cloudinit/version.py index 8560d087..f25e9145 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "20.3" +__VERSION__ = "20.4" _PACKAGED_VERSION = '@@PACKAGED_VERSION@@' FEATURES = [ -- cgit v1.2.3 From c6bcb8df28daa234686a563549681082eb3283a1 Mon Sep 17 00:00:00 2001 From: zsdc Date: Fri, 25 Dec 2020 18:36:39 +0200 Subject: T2117: Updates from 20.4 copied to resizefs_vyos This commit does not change any actually used in VyOS logic. It only copies changes from the upstream 20.4 to stay closer to the upstream code. --- cloudinit/config/cc_resizefs_vyos.py | 68 +++++----------------- .../test_handler/test_handler_resizefs_vyos.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) diff --git a/cloudinit/config/cc_resizefs_vyos.py b/cloudinit/config/cc_resizefs_vyos.py index f5555afc..f8eb84fe 100644 --- a/cloudinit/config/cc_resizefs_vyos.py +++ b/cloudinit/config/cc_resizefs_vyos.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -101,56 +98,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs_vyos.py b/tests/unittests/test_handler/test_handler_resizefs_vyos.py index d1fafff7..c18ab1ea 100644 --- a/tests/unittests/test_handler/test_handler_resizefs_vyos.py +++ b/tests/unittests/test_handler/test_handler_resizefs_vyos.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs_vyos import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3