From 1f860e5ac7ebb5b809c72d8703a0b7cb3e84ccd0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 5 Mar 2020 17:38:28 -0700 Subject: ec2: Do not fallback to IMDSv1 on EC2 (#216) The EC2 Data Source needs to handle 3 states of the Instance Metadata Service configured for a given instance: 1. HttpTokens : optional & HttpEndpoint : enabled Either IMDSv2 or IMDSv1 can be used. 2. HttpTokens : required & HttpEndpoint : enabled Calls to IMDS without a valid token (IMDSv1 or IMDSv2 with expired token) will return a 401 error. 3. HttpEndpoint : disabled The IMDS http endpoint will return a 403 error. Previous work to support IMDSv2 in cloud-init handled case 1 and case 2. This commit handles case 3 by bypassing the retry block when IMDS returns HTTP status code >= 400 on official AWS cloud platform. It shaves 2 minutes when rebooting an instance that has its IMDS http token endpoint disabled but creates some inconsistencies. An instance that doesn't set "manual_cache_clean" to "True" will have its /var/lib/cloud/instance symlink removed altogether after it has failed to find a datasource. --- tests/unittests/test_datasource/test_ec2.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'tests/unittests/test_datasource') diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 2a96122f..78e82c7e 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -3,6 +3,7 @@ import copy import httpretty import json +import requests from unittest import mock from cloudinit import helpers @@ -200,6 +201,7 @@ def register_mock_metaserver(base_url, data): class TestEc2(test_helpers.HttprettyTestCase): with_logs = True + maxDiff = None valid_platform_data = { 'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412', @@ -429,6 +431,52 @@ class TestEc2(test_helpers.HttprettyTestCase): self.assertTrue(ds.get_data()) self.assertFalse(ds.is_classic_instance()) + def test_aws_inaccessible_imds_service_fails_with_retries(self): + """Inaccessibility of http://169.254.169.254 are retried.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=None) + + conn_error = requests.exceptions.ConnectionError( + '[Errno 113] no route to host') + + mock_success = mock.MagicMock(contents=b'fakesuccess') + mock_success.ok.return_value = True + + with mock.patch('cloudinit.url_helper.readurl') as m_readurl: + m_readurl.side_effect = (conn_error, conn_error, mock_success) + with mock.patch('cloudinit.url_helper.time.sleep'): + self.assertTrue(ds.wait_for_metadata_service()) + + # Just one /latest/api/token request + self.assertEqual(3, len(m_readurl.call_args_list)) + for readurl_call in m_readurl.call_args_list: + self.assertIn('latest/api/token', readurl_call[0][0]) + + def test_aws_token_403_fails_without_retries(self): + """Verify that 403s fetching AWS tokens are not retried.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=None) + token_url = self.data_url('latest', data_item='api/token') + httpretty.register_uri(httpretty.PUT, token_url, body={}, status=403) + self.assertFalse(ds.get_data()) + # Just one /latest/api/token request + logs = self.logs.getvalue() + failed_put_log = '"PUT /latest/api/token HTTP/1.1" 403 0' + expected_logs = [ + 'WARNING: Ec2 IMDS endpoint returned a 403 error. HTTP endpoint is' + ' disabled. Aborting.', + "WARNING: IMDS's HTTP endpoint is probably disabled", + failed_put_log + ] + for log in expected_logs: + self.assertIn(log, logs) + self.assertEqual( + 1, len([l for l in logs.splitlines() if failed_put_log in l])) + def test_aws_token_redacted(self): """Verify that aws tokens are redacted when logged.""" ds = self._setup_ds( -- cgit v1.2.3 From 71af48df3514ca831c90b77dc71ba0a121dec401 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 10 Mar 2020 08:22:22 -0600 Subject: instance-data: add cloud-init merged_cfg and sys_info keys to json (#214) Cloud-config userdata provided as jinja templates are now distro, platform and merged cloud config aware. The cloud-init query command will also surface this config data. Now users can selectively render portions of cloud-config based on: * distro name, version, release * python version * merged cloud config values * machine platform * kernel To support template handling of this config, add new top-level keys to /run/cloud-init/instance-data.json. The new 'merged_cfg' key represents merged cloud config from /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/*. The new 'sys_info' key which captures distro and platform info from cloudinit.util.system_info. Cloud config userdata templates can render conditional content based on these additional environmental checks such as the following simple example: ``` ## template: jinja #cloud-config runcmd: {% if distro == 'opensuse' %} - sh /custom-setup-sles {% elif distro == 'centos' %} - sh /custom-setup-centos {% elif distro == 'debian' %} - sh /custom-setup-debian {% endif %} ``` To see all values: sudo cloud-init query --all Any keys added to the standardized v1 keys are guaranteed to not change or drop on future released of cloud-init. 'v1' keys will be retained for backward-compatibility even if a new standardized 'v2' set of keys are introduced The following standardized v1 keys are added: * distro, distro_release, distro_version, kernel_version, machine, python_version, system_platform, variant LP: #1865969 --- cloudinit/sources/__init__.py | 43 ++- cloudinit/sources/tests/test_init.py | 98 +++++- cloudinit/util.py | 2 +- doc/rtd/topics/instancedata.rst | 360 +++++++++++++++------ tests/cloud_tests/testcases/base.py | 55 +++- tests/unittests/test_datasource/test_cloudsigma.py | 6 +- 6 files changed, 426 insertions(+), 138 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 805d803d..a6e6d202 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -89,26 +89,26 @@ def process_instance_metadata(metadata, key_path='', sensitive_keys=()): @return Dict copy of processed metadata. """ md_copy = copy.deepcopy(metadata) - md_copy['base64_encoded_keys'] = [] - md_copy['sensitive_keys'] = [] + base64_encoded_keys = [] + sens_keys = [] for key, val in metadata.items(): if key_path: sub_key_path = key_path + '/' + key else: sub_key_path = key if key in sensitive_keys or sub_key_path in sensitive_keys: - md_copy['sensitive_keys'].append(sub_key_path) + sens_keys.append(sub_key_path) if isinstance(val, str) and val.startswith('ci-b64:'): - md_copy['base64_encoded_keys'].append(sub_key_path) + base64_encoded_keys.append(sub_key_path) md_copy[key] = val.replace('ci-b64:', '') if isinstance(val, dict): return_val = process_instance_metadata( val, sub_key_path, sensitive_keys) - md_copy['base64_encoded_keys'].extend( - return_val.pop('base64_encoded_keys')) - md_copy['sensitive_keys'].extend( - return_val.pop('sensitive_keys')) + base64_encoded_keys.extend(return_val.pop('base64_encoded_keys')) + sens_keys.extend(return_val.pop('sensitive_keys')) md_copy[key] = return_val + md_copy['base64_encoded_keys'] = sorted(base64_encoded_keys) + md_copy['sensitive_keys'] = sorted(sens_keys) return md_copy @@ -193,7 +193,7 @@ class DataSource(metaclass=abc.ABCMeta): # N-tuple of keypaths or keynames redact from instance-data.json for # non-root users - sensitive_metadata_keys = ('security-credentials',) + sensitive_metadata_keys = ('merged_cfg', 'security-credentials',) def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg @@ -218,14 +218,15 @@ class DataSource(metaclass=abc.ABCMeta): def __str__(self): return type_utils.obj_name(self) - def _get_standardized_metadata(self): + def _get_standardized_metadata(self, instance_data): """Return a dictionary of standardized metadata keys.""" local_hostname = self.get_hostname() instance_id = self.get_instance_id() availability_zone = self.availability_zone # In the event of upgrade from existing cloudinit, pickled datasource # will not contain these new class attributes. So we need to recrawl - # metadata to discover that content. + # metadata to discover that content + sysinfo = instance_data["sys_info"] return { 'v1': { '_beta_keys': ['subplatform'], @@ -233,14 +234,22 @@ class DataSource(metaclass=abc.ABCMeta): 'availability_zone': availability_zone, 'cloud-name': self.cloud_name, 'cloud_name': self.cloud_name, + 'distro': sysinfo["dist"][0], + 'distro_version': sysinfo["dist"][1], + 'distro_release': sysinfo["dist"][2], 'platform': self.platform_type, 'public_ssh_keys': self.get_public_ssh_keys(), + 'python_version': sysinfo["python"], 'instance-id': instance_id, 'instance_id': instance_id, + 'kernel_release': sysinfo["uname"][2], 'local-hostname': local_hostname, 'local_hostname': local_hostname, + 'machine': sysinfo["uname"][4], 'region': self.region, - 'subplatform': self.subplatform}} + 'subplatform': self.subplatform, + 'system_platform': sysinfo["platform"], + 'variant': sysinfo["variant"]}} def clear_cached_attrs(self, attr_defaults=()): """Reset any cached metadata attributes to datasource defaults. @@ -299,9 +308,15 @@ class DataSource(metaclass=abc.ABCMeta): ec2_metadata = getattr(self, 'ec2_metadata') if ec2_metadata != UNSET: instance_data['ds']['ec2_metadata'] = ec2_metadata - instance_data.update( - self._get_standardized_metadata()) instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT + # Add merged cloud.cfg and sys info for jinja templates and cli query + instance_data['merged_cfg'] = copy.deepcopy(self.sys_cfg) + instance_data['merged_cfg']['_doc'] = ( + 'Merged cloud-init system config from /etc/cloud/cloud.cfg and' + ' /etc/cloud/cloud.cfg.d/') + instance_data['sys_info'] = util.system_info() + instance_data.update( + self._get_standardized_metadata(instance_data)) try: # Process content base64encoding unserializable values content = util.json_dumps(instance_data) diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 6db127e7..541cbbeb 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -55,6 +55,7 @@ class InvalidDataSourceTestSubclassNet(DataSource): class TestDataSource(CiTestCase): with_logs = True + maxDiff = None def setUp(self): super(TestDataSource, self).setUp() @@ -288,27 +289,47 @@ class TestDataSource(CiTestCase): tmp = self.tmp_dir() datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, Paths({'run_dir': tmp})) - datasource.get_data() + sys_info = { + "python": "3.7", + "platform": + "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal", + "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah", + "x86_64"], + "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} + with mock.patch("cloudinit.util.system_info", return_value=sys_info): + datasource.get_data() json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) expected = { 'base64_encoded_keys': [], - 'sensitive_keys': [], + 'merged_cfg': REDACT_SENSITIVE_VALUE, + 'sensitive_keys': ['merged_cfg'], + 'sys_info': sys_info, 'v1': { '_beta_keys': ['subplatform'], 'availability-zone': 'myaz', 'availability_zone': 'myaz', 'cloud-name': 'subclasscloudname', 'cloud_name': 'subclasscloudname', + 'distro': 'ubuntu', + 'distro_release': 'focal', + 'distro_version': '20.04', 'instance-id': 'iid-datasource', 'instance_id': 'iid-datasource', 'local-hostname': 'test-subclass-hostname', 'local_hostname': 'test-subclass-hostname', + 'kernel_release': '5.4.0-24-generic', + 'machine': 'x86_64', 'platform': 'mytestsubclass', 'public_ssh_keys': [], + 'python_version': '3.7', 'region': 'myregion', - 'subplatform': 'unknown'}, + 'system_platform': + 'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal', + 'subplatform': 'unknown', + 'variant': 'ubuntu'}, 'ds': { + '_doc': EXPERIMENTAL_TEXT, 'meta_data': {'availability_zone': 'myaz', 'local-hostname': 'test-subclass-hostname', @@ -329,28 +350,49 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertEqual( - ('security-credentials',), datasource.sensitive_metadata_keys) - datasource.get_data() + self.assertItemsEqual( + ('merged_cfg', 'security-credentials',), + datasource.sensitive_metadata_keys) + sys_info = { + "python": "3.7", + "platform": + "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal", + "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah", + "x86_64"], + "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} + with mock.patch("cloudinit.util.system_info", return_value=sys_info): + datasource.get_data() json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) redacted = util.load_json(util.load_file(json_file)) expected = { 'base64_encoded_keys': [], - 'sensitive_keys': ['ds/meta_data/some/security-credentials'], + 'merged_cfg': REDACT_SENSITIVE_VALUE, + 'sensitive_keys': [ + 'ds/meta_data/some/security-credentials', 'merged_cfg'], + 'sys_info': sys_info, 'v1': { '_beta_keys': ['subplatform'], 'availability-zone': 'myaz', 'availability_zone': 'myaz', 'cloud-name': 'subclasscloudname', 'cloud_name': 'subclasscloudname', + 'distro': 'ubuntu', + 'distro_release': 'focal', + 'distro_version': '20.04', 'instance-id': 'iid-datasource', 'instance_id': 'iid-datasource', 'local-hostname': 'test-subclass-hostname', 'local_hostname': 'test-subclass-hostname', + 'kernel_release': '5.4.0-24-generic', + 'machine': 'x86_64', 'platform': 'mytestsubclass', 'public_ssh_keys': [], + 'python_version': '3.7', 'region': 'myregion', - 'subplatform': 'unknown'}, + 'system_platform': + 'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal', + 'subplatform': 'unknown', + 'variant': 'ubuntu'}, 'ds': { '_doc': EXPERIMENTAL_TEXT, 'meta_data': { @@ -359,7 +401,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}} } - self.assertEqual(expected, redacted) + self.assertItemsEqual(expected, redacted) file_stat = os.stat(json_file) self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) @@ -376,28 +418,54 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertEqual( - ('security-credentials',), datasource.sensitive_metadata_keys) - datasource.get_data() + sys_info = { + "python": "3.7", + "platform": + "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal", + "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah", + "x86_64"], + "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} + + self.assertItemsEqual( + ('merged_cfg', 'security-credentials',), + datasource.sensitive_metadata_keys) + with mock.patch("cloudinit.util.system_info", return_value=sys_info): + datasource.get_data() sensitive_json_file = self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, tmp) content = util.load_file(sensitive_json_file) expected = { 'base64_encoded_keys': [], - 'sensitive_keys': ['ds/meta_data/some/security-credentials'], + 'merged_cfg': { + '_doc': ( + 'Merged cloud-init system config from ' + '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'), + 'datasource': {'_undef': {'key1': False}}}, + 'sensitive_keys': [ + 'ds/meta_data/some/security-credentials', 'merged_cfg'], + 'sys_info': sys_info, 'v1': { '_beta_keys': ['subplatform'], 'availability-zone': 'myaz', 'availability_zone': 'myaz', 'cloud-name': 'subclasscloudname', 'cloud_name': 'subclasscloudname', + 'distro': 'ubuntu', + 'distro_release': 'focal', + 'distro_version': '20.04', 'instance-id': 'iid-datasource', 'instance_id': 'iid-datasource', + 'kernel_release': '5.4.0-24-generic', 'local-hostname': 'test-subclass-hostname', 'local_hostname': 'test-subclass-hostname', + 'machine': 'x86_64', 'platform': 'mytestsubclass', 'public_ssh_keys': [], + 'python_version': '3.7', 'region': 'myregion', - 'subplatform': 'unknown'}, + 'subplatform': 'unknown', + 'system_platform': + 'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal', + 'variant': 'ubuntu'}, 'ds': { '_doc': EXPERIMENTAL_TEXT, 'meta_data': { @@ -408,7 +476,7 @@ class TestDataSource(CiTestCase): 'security-credentials': {'cred1': 'sekret', 'cred2': 'othersekret'}}}} } - self.assertEqual(expected, util.load_json(content)) + self.assertItemsEqual(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) self.assertEqual(expected, util.load_json(content)) diff --git a/cloudinit/util.py b/cloudinit/util.py index c02b3d9a..132f6051 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -656,7 +656,7 @@ def system_info(): 'system': platform.system(), 'release': platform.release(), 'python': platform.python_version(), - 'uname': platform.uname(), + 'uname': list(platform.uname()), 'dist': get_linux_distro() } system = info['system'].lower() diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 4227c4fd..845098bb 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -76,6 +76,11 @@ There are three basic top-level keys: 'security sensitive'. Only the keys listed here will be redacted from instance-data.json for non-root users. +* **merged_cfg**: Merged cloud-init 'system_config' from `/etc/cloud/cloud.cfg` + and `/etc/cloud/cloud-cfg.d`. Values under this key could contain sensitive + information such as passwords, so it is included in the **sensitive-keys** + list which is only readable by root. + * **ds**: Datasource-specific metadata crawled for the specific cloud platform. It should closely represent the structure of the cloud metadata crawled. The structure of content and details provided are entirely @@ -83,6 +88,9 @@ There are three basic top-level keys: The content exposed under the 'ds' key is currently **experimental** and expected to change slightly in the upcoming cloud-init release. +* **sys_info**: Information about the underlying os, python, architecture and + kernel. This represents the data collected by `cloudinit.util.system_info`. + * **v1**: Standardized cloud-init metadata keys, these keys are guaranteed to exist on all cloud platforms. They will also retain their current behavior and format and will be carried forward even if cloud-init introduces a new @@ -117,6 +125,21 @@ Example output: - nocloud - ovf +v1.distro, v1.distro_version, v1.distro_release +----------------------------------------------- +This shall be the distro name, version and release as determined by +`cloudinit.util.get_linux_distro`. + +Example output: + +- centos, 7.5, core +- debian, 9, stretch +- freebsd, 12.0-release-p10, +- opensuse, 42.3, x86_64 +- opensuse-tumbleweed, 20180920, x86_64 +- redhat, 7.5, 'maipo' +- sles, 12.3, x86_64 +- ubuntu, 20.04, focal v1.instance_id -------------- @@ -126,6 +149,14 @@ Examples output: - i- +v1.kernel_release +----------------- +This shall be the running kernel `uname -r` + +Example output: + +- 5.3.0-1010-aws + v1.local_hostname ----------------- The internal or local hostname of the system. @@ -135,6 +166,17 @@ Examples output: - ip-10-41-41-70 - +v1.machine +---------- +This shall be the running cpu machine architecture `uname -m` + +Example output: + +- x86_64 +- i686 +- ppc64le +- s390x + v1.platform ------------- An attempt to identify the cloud platfrom instance that the system is running @@ -154,7 +196,7 @@ v1.subplatform Additional platform details describing the specific source or type of metadata used. The format of subplatform will be: -`` (`` +`` ()`` Examples output: @@ -171,6 +213,15 @@ Examples output: - ['ssh-rsa AA...', ...] +v1.python_version +----------------- +The version of python that is running cloud-init as determined by +`cloudinit.util.system_info` + +Example output: + +- 3.7.6 + v1.region --------- The physical region/data center in which the instance is deployed. @@ -192,164 +243,265 @@ Examples output: Example Output -------------- -Below is an example of ``/run/cloud-init/instance_data.json`` on an EC2 -instance: +Below is an example of ``/run/cloud-init/instance-data-sensitive.json`` on an +EC2 instance: .. sourcecode:: json { + "_beta_keys": [ + "subplatform" + ], + "availability_zone": "us-east-1b", "base64_encoded_keys": [], + "merged_cfg": { + "_doc": "Merged cloud-init system config from /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/", + "_log": [ + "[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n", + "[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n", + "[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=(\"/dev/log\", handlers.SysLogHandler.LOG_USER)\n" + ], + "cloud_config_modules": [ + "emit_upstart", + "snap", + "ssh-import-id", + "locale", + "set-passwords", + "grub-dpkg", + "apt-pipelining", + "apt-configure", + "ubuntu-advantage", + "ntp", + "timezone", + "disable-ec2-metadata", + "runcmd", + "byobu" + ], + "cloud_final_modules": [ + "package-update-upgrade-install", + "fan", + "landscape", + "lxd", + "ubuntu-drivers", + "puppet", + "chef", + "mcollective", + "salt-minion", + "rightscale_userdata", + "scripts-vendor", + "scripts-per-once", + "scripts-per-boot", + "scripts-per-instance", + "scripts-user", + "ssh-authkey-fingerprints", + "keys-to-console", + "phone-home", + "final-message", + "power-state-change" + ], + "cloud_init_modules": [ + "migrator", + "seed_random", + "bootcmd", + "write-files", + "growpart", + "resizefs", + "disk_setup", + "mounts", + "set_hostname", + "update_hostname", + "update_etc_hosts", + "ca-certs", + "rsyslog", + "users-groups", + "ssh" + ], + "datasource_list": [ + "Ec2", + "None" + ], + "def_log_file": "/var/log/cloud-init.log", + "disable_root": true, + "log_cfgs": [ + [ + "[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n", + "[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n" + ] + ], + "output": { + "all": "| tee -a /var/log/cloud-init-output.log" + }, + "preserve_hostname": false, + "syslog_fix_perms": [ + "syslog:adm", + "root:adm", + "root:wheel", + "root:root" + ], + "users": [ + "default" + ], + "vendor_data": { + "enabled": true, + "prefix": [] + } + }, + "cloud_name": "aws", + "distro": "ubuntu", + "distro_release": "focal", + "distro_version": "20.04", "ds": { "_doc": "EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.", "_metadata_api_version": "2016-09-02", "dynamic": { - "instance-identity": { + "instance_identity": { "document": { - "accountId": "437526006925", + "accountId": "329910648901", "architecture": "x86_64", - "availabilityZone": "us-east-2b", + "availabilityZone": "us-east-1b", "billingProducts": null, "devpayProductCodes": null, - "imageId": "ami-079638aae7046bdd2", - "instanceId": "i-075f088c72ad3271c", + "imageId": "ami-02e8aa396f8be3b6d", + "instanceId": "i-0929128ff2f73a2f1", "instanceType": "t2.micro", "kernelId": null, "marketplaceProductCodes": null, - "pendingTime": "2018-10-05T20:10:43Z", - "privateIp": "10.41.41.95", + "pendingTime": "2020-02-27T20:46:18Z", + "privateIp": "172.31.81.43", "ramdiskId": null, - "region": "us-east-2", + "region": "us-east-1", "version": "2017-09-30" }, "pkcs7": [ - "MIAGCSqGSIb3DQEHAqCAMIACAQExCzAJBgUrDgMCGgUAMIAGCSqGSIb3DQEHAaCAJIAEggHbewog", - "ICJkZXZwYXlQcm9kdWN0Q29kZXMiIDogbnVsbCwKICAibWFya2V0cGxhY2VQcm9kdWN0Q29kZXMi", - "IDogbnVsbCwKICAicHJpdmF0ZUlwIiA6ICIxMC40MS40MS45NSIsCiAgInZlcnNpb24iIDogIjIw", - "MTctMDktMzAiLAogICJpbnN0YW5jZUlkIiA6ICJpLTA3NWYwODhjNzJhZDMyNzFjIiwKICAiYmls", - "bGluZ1Byb2R1Y3RzIiA6IG51bGwsCiAgImluc3RhbmNlVHlwZSIgOiAidDIubWljcm8iLAogICJh", - "Y2NvdW50SWQiIDogIjQzNzUyNjAwNjkyNSIsCiAgImF2YWlsYWJpbGl0eVpvbmUiIDogInVzLWVh", - "c3QtMmIiLAogICJrZXJuZWxJZCIgOiBudWxsLAogICJyYW1kaXNrSWQiIDogbnVsbCwKICAiYXJj", - "aGl0ZWN0dXJlIiA6ICJ4ODZfNjQiLAogICJpbWFnZUlkIiA6ICJhbWktMDc5NjM4YWFlNzA0NmJk", - "ZDIiLAogICJwZW5kaW5nVGltZSIgOiAiMjAxOC0xMC0wNVQyMDoxMDo0M1oiLAogICJyZWdpb24i", - "IDogInVzLWVhc3QtMiIKfQAAAAAAADGCARcwggETAgEBMGkwXDELMAkGA1UEBhMCVVMxGTAXBgNV", - "BAgTEFdhc2hpbmd0b24gU3RhdGUxEDAOBgNVBAcTB1NlYXR0bGUxIDAeBgNVBAoTF0FtYXpvbiBX", - "ZWIgU2VydmljZXMgTExDAgkAlrpI2eVeGmcwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkq", - "hkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTE4MTAwNTIwMTA0OFowIwYJKoZIhvcNAQkEMRYEFK0k", - "Tz6n1A8/zU1AzFj0riNQORw2MAkGByqGSM44BAMELjAsAhRNrr174y98grPBVXUforN/6wZp8AIU", - "JLZBkrB2GJA8A4WJ1okq++jSrBIAAAAAAAA=" + "MIAGCSqGSIb3DQ...", + "REDACTED", + "AhQUgq0iPWqPTVnT96tZE6L1XjjLHQAAAAAAAA==" ], "rsa2048": [ - "MIAGCSqGSIb3DQEHAqCAMIACAQExDzANBglghkgBZQMEAgEFADCABgkqhkiG9w0BBwGggCSABIIB", - "23sKICAiZGV2cGF5UHJvZHVjdENvZGVzIiA6IG51bGwsCiAgIm1hcmtldHBsYWNlUHJvZHVjdENv", - "ZGVzIiA6IG51bGwsCiAgInByaXZhdGVJcCIgOiAiMTAuNDEuNDEuOTUiLAogICJ2ZXJzaW9uIiA6", - "ICIyMDE3LTA5LTMwIiwKICAiaW5zdGFuY2VJZCIgOiAiaS0wNzVmMDg4YzcyYWQzMjcxYyIsCiAg", - "ImJpbGxpbmdQcm9kdWN0cyIgOiBudWxsLAogICJpbnN0YW5jZVR5cGUiIDogInQyLm1pY3JvIiwK", - "ICAiYWNjb3VudElkIiA6ICI0Mzc1MjYwMDY5MjUiLAogICJhdmFpbGFiaWxpdHlab25lIiA6ICJ1", - "cy1lYXN0LTJiIiwKICAia2VybmVsSWQiIDogbnVsbCwKICAicmFtZGlza0lkIiA6IG51bGwsCiAg", - "ImFyY2hpdGVjdHVyZSIgOiAieDg2XzY0IiwKICAiaW1hZ2VJZCIgOiAiYW1pLTA3OTYzOGFhZTcw", - "NDZiZGQyIiwKICAicGVuZGluZ1RpbWUiIDogIjIwMTgtMTAtMDVUMjA6MTA6NDNaIiwKICAicmVn", - "aW9uIiA6ICJ1cy1lYXN0LTIiCn0AAAAAAAAxggH/MIIB+wIBATBpMFwxCzAJBgNVBAYTAlVTMRkw", - "FwYDVQQIExBXYXNoaW5ndG9uIFN0YXRlMRAwDgYDVQQHEwdTZWF0dGxlMSAwHgYDVQQKExdBbWF6", - "b24gV2ViIFNlcnZpY2VzIExMQwIJAM07oeX4xevdMA0GCWCGSAFlAwQCAQUAoGkwGAYJKoZIhvcN", - "AQkDMQsGCSqGSIb3DQEHATAcBgkqhkiG9w0BCQUxDxcNMTgxMDA1MjAxMDQ4WjAvBgkqhkiG9w0B", - "CQQxIgQgkYz0pZk3zJKBi4KP4egeOKJl/UYwu5UdE7id74pmPwMwDQYJKoZIhvcNAQEBBQAEggEA", - "dC3uIGGNul1OC1mJKSH3XoBWsYH20J/xhIdftYBoXHGf2BSFsrs9ZscXd2rKAKea4pSPOZEYMXgz", - "lPuT7W0WU89N3ZKviy/ReMSRjmI/jJmsY1lea6mlgcsJXreBXFMYucZvyeWGHdnCjamoKWXkmZlM", - "mSB1gshWy8Y7DzoKviYPQZi5aI54XK2Upt4kGme1tH1NI2Cq+hM4K+adxTbNhS3uzvWaWzMklUuU", - "QHX2GMmjAVRVc8vnA8IAsBCJJp+gFgYzi09IK+cwNgCFFPADoG6jbMHHf4sLB3MUGpiA+G9JlCnM", - "fmkjI2pNRB8spc0k4UG4egqLrqCz67WuK38tjwAAAAAAAA==" + "MIAGCSqGSIb...", + "REDACTED", + "clYQvuE45xXm7Yreg3QtQbrP//owl1eZHj6s350AAAAAAAA=" ], "signature": [ - "Tsw6h+V3WnxrNVSXBYIOs1V4j95YR1mLPPH45XnhX0/Ei3waJqf7/7EEKGYP1Cr4PTYEULtZ7Mvf", - "+xJpM50Ivs2bdF7o0c4vnplRWe3f06NI9pv50dr110j/wNzP4MZ1pLhJCqubQOaaBTF3LFutgRrt", - "r4B0mN3p7EcqD8G+ll0=" + "dA+QV+LLCWCRNddnrKleYmh2GvYo+t8urDkdgmDSsPi", + "REDACTED", + "kDT4ygyJLFkd3b4qjAs=" ] } }, - "meta-data": { - "ami-id": "ami-079638aae7046bdd2", - "ami-launch-index": "0", - "ami-manifest-path": "(unknown)", - "block-device-mapping": { + "meta_data": { + "ami_id": "ami-02e8aa396f8be3b6d", + "ami_launch_index": "0", + "ami_manifest_path": "(unknown)", + "block_device_mapping": { "ami": "/dev/sda1", - "ephemeral0": "sdb", - "ephemeral1": "sdc", "root": "/dev/sda1" }, - "hostname": "ip-10-41-41-95.us-east-2.compute.internal", - "instance-action": "none", - "instance-id": "i-075f088c72ad3271c", - "instance-type": "t2.micro", - "local-hostname": "ip-10-41-41-95.us-east-2.compute.internal", - "local-ipv4": "10.41.41.95", - "mac": "06:74:8f:39:cd:a6", + "hostname": "ip-172-31-81-43.ec2.internal", + "instance_action": "none", + "instance_id": "i-0929128ff2f73a2f1", + "instance_type": "t2.micro", + "local_hostname": "ip-172-31-81-43.ec2.internal", + "local_ipv4": "172.31.81.43", + "mac": "12:7e:c9:93:29:af", "metrics": { "vhostmd": "" }, "network": { "interfaces": { "macs": { - "06:74:8f:39:cd:a6": { - "device-number": "0", - "interface-id": "eni-052058bbd7831eaae", - "ipv4-associations": { - "18.218.221.122": "10.41.41.95" - }, - "local-hostname": "ip-10-41-41-95.us-east-2.compute.internal", - "local-ipv4s": "10.41.41.95", - "mac": "06:74:8f:39:cd:a6", - "owner-id": "437526006925", - "public-hostname": "ec2-18-218-221-122.us-east-2.compute.amazonaws.com", - "public-ipv4s": "18.218.221.122", - "security-group-ids": "sg-828247e9", - "security-groups": "Cloud-init integration test secgroup", - "subnet-id": "subnet-282f3053", - "subnet-ipv4-cidr-block": "10.41.41.0/24", - "subnet-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/64", - "vpc-id": "vpc-252ef24d", - "vpc-ipv4-cidr-block": "10.41.0.0/16", - "vpc-ipv4-cidr-blocks": "10.41.0.0/16", - "vpc-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/56" - } + "12:7e:c9:93:29:af": { + "device_number": "0", + "interface_id": "eni-0c07a0474339b801d", + "ipv4_associations": { + "3.89.187.177": "172.31.81.43" + }, + "local_hostname": "ip-172-31-81-43.ec2.internal", + "local_ipv4s": "172.31.81.43", + "mac": "12:7e:c9:93:29:af", + "owner_id": "329910648901", + "public_hostname": "ec2-3-89-187-177.compute-1.amazonaws.com", + "public_ipv4s": "3.89.187.177", + "security_group_ids": "sg-0100038b68aa79986", + "security_groups": "launch-wizard-3", + "subnet_id": "subnet-04e2d12a", + "subnet_ipv4_cidr_block": "172.31.80.0/20", + "vpc_id": "vpc-210b4b5b", + "vpc_ipv4_cidr_block": "172.31.0.0/16", + "vpc_ipv4_cidr_blocks": "172.31.0.0/16" + } } } }, "placement": { - "availability-zone": "us-east-2b" + "availability_zone": "us-east-1b" }, "profile": "default-hvm", - "public-hostname": "ec2-18-218-221-122.us-east-2.compute.amazonaws.com", - "public-ipv4": "18.218.221.122", - "public-keys": { - "cloud-init-integration": [ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB cloud-init-integration" - ] - }, - "reservation-id": "r-0594a20e31f6cfe46", - "security-groups": "Cloud-init integration test secgroup", + "public_hostname": "ec2-3-89-187-177.compute-1.amazonaws.com", + "public_ipv4": "3.89.187.177", + "reservation_id": "r-0c481643d15766a02", + "security_groups": "launch-wizard-3", "services": { "domain": "amazonaws.com", "partition": "aws" } } }, + "instance_id": "i-0929128ff2f73a2f1", + "kernel_release": "5.3.0-1010-aws", + "local_hostname": "ip-172-31-81-43", + "machine": "x86_64", + "platform": "ec2", + "public_ssh_keys": [], + "python_version": "3.7.6", + "region": "us-east-1", "sensitive_keys": [], + "subplatform": "metadata (http://169.254.169.254)", + "sys_info": { + "dist": [ + "ubuntu", + "20.04", + "focal" + ], + "platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal", + "python": "3.7.6", + "release": "5.3.0-1010-aws", + "system": "Linux", + "uname": [ + "Linux", + "ip-172-31-81-43", + "5.3.0-1010-aws", + "#11-Ubuntu SMP Thu Jan 16 07:59:32 UTC 2020", + "x86_64", + "x86_64" + ], + "variant": "ubuntu" + }, + "system_platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal", + "userdata": "#cloud-config\nssh_import_id: []\n...", "v1": { "_beta_keys": [ "subplatform" ], - "availability-zone": "us-east-2b", - "availability_zone": "us-east-2b", + "availability_zone": "us-east-1b", "cloud_name": "aws", - "instance_id": "i-075f088c72ad3271c", - "local_hostname": "ip-10-41-41-95", + "distro": "ubuntu", + "distro_release": "focal", + "distro_version": "20.04", + "instance_id": "i-0929128ff2f73a2f1", + "kernel": "5.3.0-1010-aws", + "local_hostname": "ip-172-31-81-43", + "machine": "x86_64", "platform": "ec2", - "public_ssh_keys": [ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB cloud-init-integration" - ], - "region": "us-east-2", - "subplatform": "metadata (http://169.254.169.254)" - } + "public_ssh_keys": [], + "python": "3.7.6", + "region": "us-east-1", + "subplatform": "metadata (http://169.254.169.254)", + "system_platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal", + "variant": "ubuntu" + }, + "variant": "ubuntu", + "vendordata": "" } diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index fd12d87b..5976e234 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -172,9 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual( - [], - instance_data['base64_encoded_keys']) + self.assertItemsEqual(['ci_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -201,6 +199,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIn('i-', v1_data['instance_id']) self.assertIn('ip-', v1_data['local_hostname']) self.assertIsNotNone(v1_data['region'], 'expected ec2 region') + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+-aws', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) def test_instance_data_json_lxd(self): """Validate instance-data.json content by lxd platform. @@ -237,6 +252,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) def test_instance_data_json_kvm(self): """Validate instance-data.json content by nocloud-kvm platform. @@ -278,6 +310,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) class PasswordListTest(CloudTestCase): diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index d62d542b..7aa3b1d1 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -3,6 +3,7 @@ import copy from cloudinit.cs_utils import Cepko +from cloudinit import distros from cloudinit import helpers from cloudinit import sources from cloudinit.sources import DataSourceCloudSigma @@ -47,8 +48,11 @@ class DataSourceCloudSigmaTest(test_helpers.CiTestCase): self.paths = helpers.Paths({'run_dir': self.tmp_dir()}) self.add_patch(DS_PATH + '.is_running_in_cloudsigma', "m_is_container", return_value=True) + + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=self.paths) self.datasource = DataSourceCloudSigma.DataSourceCloudSigma( - "", "", paths=self.paths) + sys_cfg={}, distro=distro, paths=self.paths) self.datasource.cepko = CepkoMock(SERVER_CONTEXT) def test_get_hostname(self): -- cgit v1.2.3 From 6600c642af3817fe5e0170cb7b4eeac4be3c60eb Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 18 Mar 2020 13:33:37 -0600 Subject: ec2: render network on all NICs and add secondary IPs as static (#114) Add support for rendering secondary static IPv4/IPv6 addresses on any NIC attached to the machine. In order to see secondary IP addresses in Ec2 IMDS network config, cloud-init now reads metadata version 2018-09-24. Metadata services which do not support the Ec2 API version will not get secondary IP addresses configured. In order to discover secondary IP address config, cloud-init now relies on metadata API Parse local-ipv4s, ipv6s, subnet-ipv4-cidr-block and subnet-ipv6-cidr-block metadata keys to determine additional IPs and appropriate subnet prefix to set for a nic. Also add the datasource config option apply_full_imds_netork_config which defaults to true to allow cloud-init to automatically configure secondary IP addresses. Setting this option to false will tell cloud-init to avoid setting up secondary IP addresses. Also in this branch: - Shift Ec2 datasource to emit network config v2 instead of v1. LP: #1866930 --- cloudinit/sources/DataSourceEc2.py | 116 ++++++++-- doc/rtd/topics/datasources/ec2.rst | 19 ++ tests/unittests/test_datasource/test_ec2.py | 329 ++++++++++++++++++++++------ 3 files changed, 379 insertions(+), 85 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 8f0d73bb..4203c3a6 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -62,7 +62,7 @@ class DataSourceEc2(sources.DataSource): # Priority ordered list of additional metadata versions which will be tried # for extended metadata content. IPv6 support comes in 2016-09-02 - extended_metadata_versions = ['2016-09-02'] + extended_metadata_versions = ['2018-09-24', '2016-09-02'] # Setup read_url parameters per get_url_params. url_max_wait = 120 @@ -405,13 +405,16 @@ class DataSourceEc2(sources.DataSource): logfunc=LOG.debug, msg='Re-crawl of metadata service', func=self.get_data) - # Limit network configuration to only the primary/fallback nic iface = self.fallback_interface - macs_to_nics = {net.get_interface_mac(iface): iface} net_md = self.metadata.get('network') if isinstance(net_md, dict): + # SRU_BLOCKER: xenial, bionic and eoan should default + # apply_full_imds_network_config to False to retain original + # behavior on those releases. result = convert_ec2_metadata_network_config( - net_md, macs_to_nics=macs_to_nics, fallback_nic=iface) + net_md, fallback_nic=iface, + full_network_config=util.get_cfg_option_bool( + self.ds_cfg, 'apply_full_imds_network_config', True)) # RELEASE_BLOCKER: xenial should drop the below if statement, # because the issue being addressed doesn't exist pre-netplan. @@ -719,9 +722,10 @@ def _collect_platform_data(): return data -def convert_ec2_metadata_network_config(network_md, macs_to_nics=None, - fallback_nic=None): - """Convert ec2 metadata to network config version 1 data dict. +def convert_ec2_metadata_network_config( + network_md, macs_to_nics=None, fallback_nic=None, + full_network_config=True): + """Convert ec2 metadata to network config version 2 data dict. @param: network_md: 'network' portion of EC2 metadata. generally formed as {"interfaces": {"macs": {}} where @@ -731,28 +735,104 @@ def convert_ec2_metadata_network_config(network_md, macs_to_nics=None, not provided, get_interfaces_by_mac is called to get it from the OS. @param: fallback_nic: Optionally provide the primary nic interface name. This nic will be guaranteed to minimally have a dhcp4 configuration. + @param: full_network_config: Boolean set True to configure all networking + presented by IMDS. This includes rendering secondary IPv4 and IPv6 + addresses on all NICs and rendering network config on secondary NICs. + If False, only the primary nic will be configured and only with dhcp + (IPv4/IPv6). - @return A dict of network config version 1 based on the metadata and macs. + @return A dict of network config version 2 based on the metadata and macs. """ - netcfg = {'version': 1, 'config': []} + netcfg = {'version': 2, 'ethernets': {}} if not macs_to_nics: macs_to_nics = net.get_interfaces_by_mac() macs_metadata = network_md['interfaces']['macs'] - for mac, nic_name in macs_to_nics.items(): + + if not full_network_config: + for mac, nic_name in macs_to_nics.items(): + if nic_name == fallback_nic: + break + dev_config = {'dhcp4': True, + 'dhcp6': False, + 'match': {'macaddress': mac.lower()}, + 'set-name': nic_name} + nic_metadata = macs_metadata.get(mac) + if nic_metadata.get('ipv6s'): # Any IPv6 addresses configured + dev_config['dhcp6'] = True + netcfg['ethernets'][nic_name] = dev_config + return netcfg + # Apply network config for all nics and any secondary IPv4/v6 addresses + nic_idx = 1 + for mac, nic_name in sorted(macs_to_nics.items()): nic_metadata = macs_metadata.get(mac) if not nic_metadata: continue # Not a physical nic represented in metadata - nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []} - nic_cfg['mac_address'] = mac - if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or - nic_metadata.get('local-ipv4s')): - nic_cfg['subnets'].append({'type': 'dhcp4'}) - if nic_metadata.get('ipv6s'): - nic_cfg['subnets'].append({'type': 'dhcp6'}) - netcfg['config'].append(nic_cfg) + dhcp_override = {'route-metric': nic_idx * 100} + nic_idx += 1 + dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, + 'dhcp6': False, + 'match': {'macaddress': mac.lower()}, + 'set-name': nic_name} + if nic_metadata.get('ipv6s'): # Any IPv6 addresses configured + dev_config['dhcp6'] = True + dev_config['dhcp6-overrides'] = dhcp_override + dev_config['addresses'] = get_secondary_addresses(nic_metadata, mac) + if not dev_config['addresses']: + dev_config.pop('addresses') # Since we found none configured + netcfg['ethernets'][nic_name] = dev_config + # Remove route-metric dhcp overrides if only one nic configured + if len(netcfg['ethernets']) == 1: + for nic_name in netcfg['ethernets'].keys(): + netcfg['ethernets'][nic_name].pop('dhcp4-overrides') + netcfg['ethernets'][nic_name].pop('dhcp6-overrides', None) return netcfg +def get_secondary_addresses(nic_metadata, mac): + """Parse interface-specific nic metadata and return any secondary IPs + + :return: List of secondary IPv4 or IPv6 addresses to configure on the + interface + """ + ipv4s = nic_metadata.get('local-ipv4s') + ipv6s = nic_metadata.get('ipv6s') + addresses = [] + # In version < 2018-09-24 local_ipv4s or ipv6s is a str with one IP + if bool(isinstance(ipv4s, list) and len(ipv4s) > 1): + addresses.extend( + _get_secondary_addresses( + nic_metadata, 'subnet-ipv4-cidr-block', mac, ipv4s, '24')) + if bool(isinstance(ipv6s, list) and len(ipv6s) > 1): + addresses.extend( + _get_secondary_addresses( + nic_metadata, 'subnet-ipv6-cidr-block', mac, ipv6s, '128')) + return sorted(addresses) + + +def _get_secondary_addresses(nic_metadata, cidr_key, mac, ips, default_prefix): + """Return list of IP addresses as CIDRs for secondary IPs + + The CIDR prefix will be default_prefix if cidr_key is absent or not + parseable in nic_metadata. + """ + addresses = [] + cidr = nic_metadata.get(cidr_key) + prefix = default_prefix + if not cidr or len(cidr.split('/')) != 2: + ip_type = 'ipv4' if 'ipv4' in cidr_key else 'ipv6' + LOG.warning( + 'Could not parse %s %s for mac %s. %s network' + ' config prefix defaults to /%s', + cidr_key, cidr, mac, ip_type, prefix) + else: + prefix = cidr.split('/')[1] + # We know we have > 1 ips for in metadata for this IP type + for ip in ips[1:]: + addresses.append( + '{ip}/{prefix}'.format(ip=ip, prefix=prefix)) + return addresses + + # Used to match classes to dependencies datasources = [ (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst index a90f3779..1c3a880f 100644 --- a/doc/rtd/topics/datasources/ec2.rst +++ b/doc/rtd/topics/datasources/ec2.rst @@ -42,6 +42,7 @@ Note that there are multiple versions of this data provided, cloud-init by default uses **2009-04-04** but newer versions can be supported with relative ease (newer versions have more data exposed, while maintaining backward compatibility with the previous versions). +Version **2016-09-02** is required for secondary IP address support. To see which versions are supported from your cloud provider use the following URL: @@ -80,6 +81,15 @@ The settings that may be configured are: * **timeout**: the timeout value provided to urlopen for each individual http request. This is used both when selecting a metadata_url and when crawling the metadata service. (default: 50) + * **apply_full_imds_network_config**: Boolean (default: True) to allow + cloud-init to configure any secondary NICs and secondary IPs described by + the metadata service. All network interfaces are configured with DHCP (v4) + to obtain an primary IPv4 address and route. Interfaces which have a + non-empty 'ipv6s' list will also enable DHCPv6 to obtain a primary IPv6 + address and route. The DHCP response (v4 and v6) return an IP that matches + the first element of local-ipv4s and ipv6s lists respectively. All + additional values (secondary addresses) in the static ip lists will be + added to interface. An example configuration with the default values is provided below: @@ -90,6 +100,7 @@ An example configuration with the default values is provided below: metadata_urls: ["http://169.254.169.254:80", "http://instance-data:8773"] max_wait: 120 timeout: 50 + apply_full_imds_network_config: true Notes ----- @@ -102,4 +113,12 @@ Notes The check for the instance type is performed by is_classic_instance() method. + * For EC2 instances with multiple network interfaces (NICs) attached, dhcp4 + will be enabled to obtain the primary private IPv4 address of those NICs. + Wherever dhcp4 or dhcp6 is enabled for a NIC, a dhcp route-metric will be + added with the value of `` * 100`` to ensure dhcp + routes on the primary NIC are preferred to any secondary NICs. + For example: the primary NIC will have a DHCP route-metric of 100, + the next NIC will be 200. + .. vi: textwidth=78 diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 78e82c7e..9126c676 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -113,6 +113,122 @@ DEFAULT_METADATA = { "services": {"domain": "amazonaws.com", "partition": "aws"}, } +# collected from api version 2018-09-24/ with +# python3 -c 'import json +# from cloudinit.ec2_utils import get_instance_metadata as gm +# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))' + +NIC1_MD_IPV4_IPV6_MULTI_IP = { + "device-number": "0", + "interface-id": "eni-0d6335689899ce9cc", + "ipv4-associations": { + "18.218.219.181": "172.31.44.13" + }, + "ipv6s": [ + "2600:1f16:292:100:c187:593c:4349:136", + "2600:1f16:292:100:f153:12a3:c37c:11f9", + "2600:1f16:292:100:f152:2222:3333:4444" + ], + "local-hostname": ("ip-172-31-44-13.us-east-2." + "compute.internal"), + "local-ipv4s": [ + "172.31.44.13", + "172.31.45.70" + ], + "mac": "0a:07:84:3d:6e:38", + "owner-id": "329910648901", + "public-hostname": ("ec2-18-218-219-181.us-east-2." + "compute.amazonaws.com"), + "public-ipv4s": "18.218.219.181", + "security-group-ids": "sg-0c387755222ba8d2e", + "security-groups": "launch-wizard-4", + "subnet-id": "subnet-9d7ba0d1", + "subnet-ipv4-cidr-block": "172.31.32.0/20", + "subnet_ipv6_cidr_blocks": "2600:1f16:292:100::/64", + "vpc-id": "vpc-a07f62c8", + "vpc-ipv4-cidr-block": "172.31.0.0/16", + "vpc-ipv4-cidr-blocks": "172.31.0.0/16", + "vpc_ipv6_cidr_blocks": "2600:1f16:292:100::/56" +} + +NIC2_MD = { + "device_number": "1", + "interface_id": "eni-043cdce36ded5e79f", + "local_hostname": "ip-172-31-47-221.us-east-2.compute.internal", + "local_ipv4s": "172.31.47.221", + "mac": "0a:75:69:92:e2:16", + "owner_id": "329910648901", + "security_group_ids": "sg-0d68fef37d8cc9b77", + "security_groups": "launch-wizard-17", + "subnet_id": "subnet-9d7ba0d1", + "subnet_ipv4_cidr_block": "172.31.32.0/20", + "vpc_id": "vpc-a07f62c8", + "vpc_ipv4_cidr_block": "172.31.0.0/16", + "vpc_ipv4_cidr_blocks": "172.31.0.0/16" +} + +SECONDARY_IP_METADATA_2018_09_24 = { + "ami-id": "ami-0986c2ac728528ac2", + "ami-launch-index": "0", + "ami-manifest-path": "(unknown)", + "block-device-mapping": { + "ami": "/dev/sda1", + "root": "/dev/sda1" + }, + "events": { + "maintenance": { + "history": "[]", + "scheduled": "[]" + } + }, + "hostname": "ip-172-31-44-13.us-east-2.compute.internal", + "identity-credentials": { + "ec2": { + "info": { + "AccountId": "329910648901", + "Code": "Success", + "LastUpdated": "2019-07-06T14:22:56Z" + } + } + }, + "instance-action": "none", + "instance-id": "i-069e01e8cc43732f8", + "instance-type": "t2.micro", + "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal", + "local-ipv4": "172.31.44.13", + "mac": "0a:07:84:3d:6e:38", + "metrics": { + "vhostmd": "" + }, + "network": { + "interfaces": { + "macs": { + "0a:07:84:3d:6e:38": NIC1_MD_IPV4_IPV6_MULTI_IP, + } + } + }, + "placement": { + "availability-zone": "us-east-2c" + }, + "profile": "default-hvm", + "public-hostname": ( + "ec2-18-218-219-181.us-east-2.compute.amazonaws.com"), + "public-ipv4": "18.218.219.181", + "public-keys": { + "yourkeyname,e": [ + "ssh-rsa AAAAW...DZ yourkeyname" + ] + }, + "reservation-id": "r-09b4917135cdd33be", + "security-groups": "launch-wizard-4", + "services": { + "domain": "amazonaws.com", + "partition": "aws" + } +} + +M_PATH_NET = 'cloudinit.sources.DataSourceEc2.net.' + def _register_ssh_keys(rfunc, base_url, keys_data): """handle ssh key inconsistencies. @@ -267,30 +383,23 @@ class TestEc2(test_helpers.HttprettyTestCase): register_mock_metaserver(instance_id_url, None) return ds - def test_network_config_property_returns_version_1_network_data(self): - """network_config property returns network version 1 for metadata. - - Only one device is configured even when multiple exist in metadata. - """ + def test_network_config_property_returns_version_2_network_data(self): + """network_config property returns network version 2 for metadata""" ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md={'md': DEFAULT_METADATA}) - find_fallback_path = ( - 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic') + find_fallback_path = M_PATH_NET + 'find_fallback_nic' with mock.patch(find_fallback_path) as m_find_fallback: m_find_fallback.return_value = 'eth9' ds.get_data() mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA - expected = {'version': 1, 'config': [ - {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}], - 'type': 'physical'}]} - patch_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') - get_interface_mac_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interface_mac') + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': True}}} + patch_path = M_PATH_NET + 'get_interfaces_by_mac' + get_interface_mac_path = M_PATH_NET + 'get_interface_mac' with mock.patch(patch_path) as m_get_interfaces_by_mac: with mock.patch(find_fallback_path) as m_find_fallback: with mock.patch(get_interface_mac_path) as m_get_mac: @@ -299,30 +408,59 @@ class TestEc2(test_helpers.HttprettyTestCase): m_get_mac.return_value = mac1 self.assertEqual(expected, ds.network_config) - def test_network_config_property_set_dhcp4_on_private_ipv4(self): - """network_config property configures dhcp4 on private ipv4 nics. + def test_network_config_property_set_dhcp4(self): + """network_config property configures dhcp4 on nics with local-ipv4s. - Only one device is configured even when multiple exist in metadata. + Only one device is configured based on get_interfaces_by_mac even when + multiple MACs exist in metadata. """ ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md={'md': DEFAULT_METADATA}) - find_fallback_path = ( - 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic') + find_fallback_path = M_PATH_NET + 'find_fallback_nic' with mock.patch(find_fallback_path) as m_find_fallback: m_find_fallback.return_value = 'eth9' ds.get_data() mac1 = '06:17:04:d7:26:0A' # IPv4 only in DEFAULT_METADATA - expected = {'version': 1, 'config': [ - {'mac_address': '06:17:04:d7:26:0A', 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}], - 'type': 'physical'}]} - patch_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') - get_interface_mac_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interface_mac') + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': False}}} + patch_path = M_PATH_NET + 'get_interfaces_by_mac' + get_interface_mac_path = M_PATH_NET + 'get_interface_mac' + with mock.patch(patch_path) as m_get_interfaces_by_mac: + with mock.patch(find_fallback_path) as m_find_fallback: + with mock.patch(get_interface_mac_path) as m_get_mac: + m_get_interfaces_by_mac.return_value = {mac1: 'eth9'} + m_find_fallback.return_value = 'eth9' + m_get_mac.return_value = mac1 + self.assertEqual(expected, ds.network_config) + + def test_network_config_property_secondary_private_ips(self): + """network_config property configures any secondary ipv4 addresses. + + Only one device is configured based on get_interfaces_by_mac even when + multiple MACs exist in metadata. + """ + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md={'md': SECONDARY_IP_METADATA_2018_09_24}) + find_fallback_path = M_PATH_NET + 'find_fallback_nic' + with mock.patch(find_fallback_path) as m_find_fallback: + m_find_fallback.return_value = 'eth9' + ds.get_data() + + mac1 = '0a:07:84:3d:6e:38' # 1 secondary IPv4 and 2 secondary IPv6 + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': mac1}, 'set-name': 'eth9', + 'addresses': ['172.31.45.70/20', + '2600:1f16:292:100:f152:2222:3333:4444/128', + '2600:1f16:292:100:f153:12a3:c37c:11f9/128'], + 'dhcp4': True, 'dhcp6': True}}} + patch_path = M_PATH_NET + 'get_interfaces_by_mac' + get_interface_mac_path = M_PATH_NET + 'get_interface_mac' with mock.patch(patch_path) as m_get_interfaces_by_mac: with mock.patch(find_fallback_path) as m_find_fallback: with mock.patch(get_interface_mac_path) as m_get_mac: @@ -358,21 +496,18 @@ class TestEc2(test_helpers.HttprettyTestCase): register_mock_metaserver( 'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA) mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA - get_interface_mac_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interface_mac') + get_interface_mac_path = M_PATH_NET + 'get_interfaces_by_mac' ds.fallback_nic = 'eth9' - with mock.patch(get_interface_mac_path) as m_get_interface_mac: - m_get_interface_mac.return_value = mac1 + with mock.patch(get_interface_mac_path) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {mac1: 'eth9'} nc = ds.network_config # Will re-crawl network metadata self.assertIsNotNone(nc) self.assertIn( 'Refreshing stale metadata from prior to upgrade', self.logs.getvalue()) - expected = {'version': 1, 'config': [ - {'mac_address': '06:17:04:d7:26:09', - 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}], - 'type': 'physical'}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': True}}} self.assertEqual(expected, ds.network_config) def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self): @@ -491,7 +626,7 @@ class TestEc2(test_helpers.HttprettyTestCase): logs_with_redacted = [log for log in all_logs if REDACT_TOK in log] logs_with_token = [log for log in all_logs if 'API-TOKEN' in log] self.assertEqual(1, len(logs_with_redacted_ttl)) - self.assertEqual(79, len(logs_with_redacted)) + self.assertEqual(81, len(logs_with_redacted)) self.assertEqual(0, len(logs_with_token)) @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -612,6 +747,44 @@ class TestEc2(test_helpers.HttprettyTestCase): self.assertIn('Crawl of metadata service took', self.logs.getvalue()) +class TestGetSecondaryAddresses(test_helpers.CiTestCase): + + mac = '06:17:04:d7:26:ff' + with_logs = True + + def test_md_with_no_secondary_addresses(self): + """Empty list is returned when nic metadata contains no secondary ip""" + self.assertEqual([], ec2.get_secondary_addresses(NIC2_MD, self.mac)) + + def test_md_with_secondary_v4_and_v6_addresses(self): + """All secondary addresses are returned from nic metadata""" + self.assertEqual( + ['172.31.45.70/20', '2600:1f16:292:100:f152:2222:3333:4444/128', + '2600:1f16:292:100:f153:12a3:c37c:11f9/128'], + ec2.get_secondary_addresses(NIC1_MD_IPV4_IPV6_MULTI_IP, self.mac)) + + def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self): + """Any invalid subnet-ipv(4|6)-cidr-block values use defaults""" + invalid_cidr_md = copy.deepcopy(NIC1_MD_IPV4_IPV6_MULTI_IP) + invalid_cidr_md['subnet-ipv4-cidr-block'] = "something-unexpected" + invalid_cidr_md['subnet-ipv6-cidr-block'] = "not/sure/what/this/is" + self.assertEqual( + ['172.31.45.70/24', '2600:1f16:292:100:f152:2222:3333:4444/128', + '2600:1f16:292:100:f153:12a3:c37c:11f9/128'], + ec2.get_secondary_addresses(invalid_cidr_md, self.mac)) + expected_logs = [ + "WARNING: Could not parse subnet-ipv4-cidr-block" + " something-unexpected for mac 06:17:04:d7:26:ff." + " ipv4 network config prefix defaults to /24", + "WARNING: Could not parse subnet-ipv6-cidr-block" + " not/sure/what/this/is for mac 06:17:04:d7:26:ff." + " ipv6 network config prefix defaults to /128" + ] + logs = self.logs.getvalue() + for log in expected_logs: + self.assertIn(log, logs) + + class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): def setUp(self): @@ -619,16 +792,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): self.mac1 = '06:17:04:d7:26:09' self.network_metadata = { 'interfaces': {'macs': { - self.mac1: {'public-ipv4s': '172.31.2.16'}}}} + self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}} def test_convert_ec2_metadata_network_config_skips_absent_macs(self): """Any mac absent from metadata is skipped by network config.""" macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'} # DE:AD:BE:EF:FF:FF represented by OS but not in metadata - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': False}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( @@ -642,15 +815,15 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): network_metadata_ipv6['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' nic1_metadata.pop('public-ipv4s') - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': True}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( network_metadata_ipv6, macs_to_nics)) - def test_convert_ec2_metadata_network_config_handles_local_dhcp4(self): + def test_convert_ec2_metadata_network_config_local_only_dhcp4(self): """Config dhcp4 when there are no public addresses in public-ipv4s.""" macs_to_nics = {self.mac1: 'eth9'} network_metadata_ipv6 = copy.deepcopy(self.network_metadata) @@ -658,9 +831,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): network_metadata_ipv6['interfaces']['macs'][self.mac1]) nic1_metadata['local-ipv4s'] = '172.3.3.15' nic1_metadata.pop('public-ipv4s') - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': False}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( @@ -675,16 +848,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): nic1_metadata['public-ipv4s'] = '' # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config. - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': False}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( network_metadata_ipv6, macs_to_nics, fallback_nic='eth9')) def test_convert_ec2_metadata_network_config_handles_local_v4_and_v6(self): - """When dhcp6 is public and dhcp4 is set to local enable both.""" + """When ipv6s and local-ipv4s are non-empty, enable dhcp6 and dhcp4.""" macs_to_nics = {self.mac1: 'eth9'} network_metadata_both = copy.deepcopy(self.network_metadata) nic1_metadata = ( @@ -692,10 +865,35 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' nic1_metadata.pop('public-ipv4s') nic1_metadata['local-ipv4s'] = '10.0.0.42' # Local ipv4 only on vpc - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': True}}} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + network_metadata_both, macs_to_nics)) + + def test_convert_ec2_metadata_network_config_handles_multiple_nics(self): + """DHCP route-metric increases on secondary NICs for IPv4 and IPv6.""" + mac2 = '06:17:04:d7:26:0a' + macs_to_nics = {self.mac1: 'eth9', mac2: 'eth10'} + network_metadata_both = copy.deepcopy(self.network_metadata) + # Add 2nd nic info + network_metadata_both['interfaces']['macs'][mac2] = NIC2_MD + nic1_metadata = ( + network_metadata_both['interfaces']['macs'][self.mac1]) + nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' + nic1_metadata.pop('public-ipv4s') # No public-ipv4 IPs in cfg + nic1_metadata['local-ipv4s'] = '10.0.0.42' # Local ipv4 only on vpc + expected = {'version': 2, 'ethernets': { + 'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp4-overrides': {'route-metric': 100}, + 'dhcp6': True, 'dhcp6-overrides': {'route-metric': 100}}, + 'eth10': { + 'match': {'macaddress': mac2}, 'set-name': 'eth10', + 'dhcp4': True, 'dhcp4-overrides': {'route-metric': 200}, + 'dhcp6': False}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( @@ -708,10 +906,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): nic1_metadata = ( network_metadata_both['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]} + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, 'set-name': 'eth9', + 'dhcp4': True, 'dhcp6': True}}} self.assertEqual( expected, ec2.convert_ec2_metadata_network_config( @@ -719,12 +916,10 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self): """Convert Ec2 Metadata calls get_interfaces_by_mac by default.""" - expected = {'version': 1, 'config': [ - {'mac_address': self.mac1, 'type': 'physical', - 'name': 'eth9', - 'subnets': [{'type': 'dhcp4'}]}]} - patch_path = ( - 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') + expected = {'version': 2, 'ethernets': {'eth9': { + 'match': {'macaddress': self.mac1}, + 'set-name': 'eth9', 'dhcp4': True, 'dhcp6': False}}} + patch_path = M_PATH_NET + 'get_interfaces_by_mac' with mock.patch(patch_path) as m_get_interfaces_by_mac: m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'} self.assertEqual( -- cgit v1.2.3 From ade47866aa2f81ab0f3baabbb1fc1e484abdb741 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Mar 2020 12:01:15 -0400 Subject: cloudinit/tests: remove unneeded with_logs configuration (#263) These classes don't use `self.logs` anywhere in their body, so we can remove the `with_logs = True` setting from them. These instances were found using astpath[0], with the following invocation: astpath "//Name[@id='with_logs' and not(ancestor::ClassDef//Attribute[@attr='logs'])]" [0] https://github.com/hchasestevens/astpath --- cloudinit/cmd/tests/test_main.py | 2 -- cloudinit/config/tests/test_disable_ec2_metadata.py | 2 -- cloudinit/net/tests/test_init.py | 6 ------ cloudinit/sources/tests/test_oracle.py | 2 -- tests/unittests/test_datasource/test_azure.py | 2 -- tests/unittests/test_datasource/test_maas.py | 1 - tests/unittests/test_handler/test_handler_locale.py | 2 -- tests/unittests/test_handler/test_handler_puppet.py | 2 -- tests/unittests/test_util.py | 1 - 9 files changed, 20 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index 384fddc6..585b3b0e 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -18,8 +18,6 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand') class TestMain(FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestMain, self).setUp() self.new_root = self.tmp_dir() diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 67646b03..823917c7 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,6 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - with_logs = True - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') def test_disable_ifconfig(self, m_subp, m_which): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 5081a337..a965699a 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -341,8 +341,6 @@ class TestGenerateFallbackConfig(CiTestCase): class TestNetFindFallBackNic(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFindFallBackNic, self).setUp() sys_mock = mock.patch('cloudinit.net.get_sys_class_path') @@ -995,8 +993,6 @@ class TestExtractPhysdevs(CiTestCase): class TestWaitForPhysdevs(CiTestCase): - with_logs = True - def setUp(self): super(TestWaitForPhysdevs, self).setUp() self.add_patch('cloudinit.net.get_interfaces_by_mac', @@ -1071,8 +1067,6 @@ class TestWaitForPhysdevs(CiTestCase): class TestNetFailOver(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFailOver, self).setUp() self.add_patch('cloudinit.net.util', 'm_util') diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py index abf3d359..316efc4f 100644 --- a/cloudinit/sources/tests/test_oracle.py +++ b/cloudinit/sources/tests/test_oracle.py @@ -588,8 +588,6 @@ class TestNetworkConfigFromOpcImds(test_helpers.CiTestCase): class TestNetworkConfigFiltersNetFailover(test_helpers.CiTestCase): - with_logs = True - def setUp(self): super(TestNetworkConfigFiltersNetFailover, self).setUp() self.add_patch(DS_PATH + '.get_interfaces_by_mac', diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a809fd87..2c6aa44d 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -383,8 +383,6 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): class TestAzureDataSource(CiTestCase): - with_logs = True - def setUp(self): super(TestAzureDataSource, self).setUp() self.tmp = self.tmp_dir() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 2a81d3f5..41b6c27b 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -158,7 +158,6 @@ class TestMAASDataSource(CiTestCase): @mock.patch("cloudinit.sources.DataSourceMAAS.url_helper.OauthUrlHelper") class TestGetOauthHelper(CiTestCase): - with_logs = True base_cfg = {'consumer_key': 'FAKE_CONSUMER_KEY', 'token_key': 'FAKE_TOKEN_KEY', 'token_secret': 'FAKE_TOKEN_SECRET', diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 2b22559f..407aa6c4 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -29,8 +29,6 @@ LOG = logging.getLogger(__name__) class TestLocale(t_help.FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestLocale, self).setUp() self.new_root = tempfile.mkdtemp() diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 1494177d..04aa7d03 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -16,8 +16,6 @@ LOG = logging.getLogger(__name__) @mock.patch('cloudinit.config.cc_puppet.os') class TestAutostartPuppet(CiTestCase): - with_logs = True - def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): """Update /etc/default/puppet to autostart if it exists.""" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 9ff17f52..5b2eaa69 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -737,7 +737,6 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.CiTestCase): - with_logs = True allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, BOGUS_COMMAND, sys.executable] -- cgit v1.2.3 From 7f9f33dbb08bacaa49244e857847400ad8a67ad4 Mon Sep 17 00:00:00 2001 From: Silvio Knizek Date: Thu, 26 Mar 2020 22:15:37 +0100 Subject: Identify SAP Converged Cloud as OpenStack add SAP Converged Cloud as cloud provider --- cloudinit/apport.py | 1 + cloudinit/sources/DataSourceOpenStack.py | 5 ++++- doc/rtd/topics/datasources/openstack.rst | 4 ++-- tests/unittests/test_datasource/test_openstack.py | 18 ++++++++++++++++++ tests/unittests/test_ds_identify.py | 10 ++++++++++ tools/ds-identify | 4 ++++ 6 files changed, 39 insertions(+), 3 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 1f2c2e7e..9bded16c 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -36,6 +36,7 @@ KNOWN_CLOUD_NAMES = [ 'OVF', 'RbxCloud - (HyperOne, Rootbox, Rubikon)', 'OpenTelekomCloud', + 'SAP Converged Cloud', 'Scaleway', 'SmartOS', 'VMware', diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 7a5e71b6..bf539091 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -29,7 +29,10 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova' DMI_PRODUCT_COMPUTE = 'OpenStack Compute' VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE] DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud' -VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM] +# See github.com/sapcc/helm-charts/blob/master/openstack/nova/values.yaml +# -> compute.defaults.vmware.smbios_asset_tag for this value +DMI_ASSET_TAG_SAPCCLOUD = 'SAP CCloud VM' +VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_SAPCCLOUD] class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): diff --git a/doc/rtd/topics/datasources/openstack.rst b/doc/rtd/topics/datasources/openstack.rst index ff817e45..ef10ce57 100644 --- a/doc/rtd/topics/datasources/openstack.rst +++ b/doc/rtd/topics/datasources/openstack.rst @@ -19,8 +19,8 @@ checks the following environment attributes as a potential OpenStack platform: * **/proc/1/environ**: Nova-lxd contains *product_name=OpenStack Nova* * **DMI product_name**: Either *Openstack Nova* or *OpenStack Compute* - * **DMI chassis_asset_tag** is *OpenTelekomCloud* or *OpenStack Nova* - (since 19.2) or *OpenStack Compute* (since 19.2) + * **DMI chassis_asset_tag** is *OpenTelekomCloud*, *SAP CCloud VM*, + *OpenStack Nova* (since 19.2) or *OpenStack Compute* (since 19.2) Configuration diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index f754556f..b534457c 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -509,6 +509,24 @@ class TestDetectOpenStack(test_helpers.CiTestCase): ds.detect_openstack(), 'Expected detect_openstack == True on OpenTelekomCloud') + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_sapccloud_chassis_asset_tag(self, m_dmi, + m_is_x86): + """Return True on OpenStack reporting SAP CCloud VM asset-tag.""" + m_is_x86.return_value = True + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'VMware Virtual Platform' # SAP CCloud uses VMware + if dmi_key == 'chassis-asset-tag': + return 'SAP CCloud VM' + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertTrue( + ds.detect_openstack(), + 'Expected detect_openstack == True on SAP CCloud VM') + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') def test_detect_openstack_oraclecloud_chassis_asset_tag(self, m_dmi, m_is_x86): diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 36d7fbbf..f0e96b44 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -447,6 +447,10 @@ class TestDsIdentify(DsIdentifyBase): """Open Telecom identification.""" self._test_ds_found('OpenStack-OpenTelekom') + def test_openstack_sap_ccloud(self): + """SAP Converged Cloud identification""" + self._test_ds_found('OpenStack-SAPCCloud') + def test_openstack_asset_tag_nova(self): """OpenStack identification via asset tag OpenStack Nova.""" self._test_ds_found('OpenStack-AssetTag-Nova') @@ -834,6 +838,12 @@ VALID_CFG = { 'files': {P_CHASSIS_ASSET_TAG: 'OpenTelekomCloud\n'}, 'mocks': [MOCK_VIRT_IS_XEN], }, + 'OpenStack-SAPCCloud': { + # SAP CCloud hosts use OpenStack on VMware + 'ds': 'OpenStack', + 'files': {P_CHASSIS_ASSET_TAG: 'SAP CCloud VM\n'}, + 'mocks': [MOCK_VIRT_IS_VMWARE], + }, 'OpenStack-AssetTag-Nova': { # VMware vSphere can't modify product-name, LP: #1669875 'ds': 'OpenStack', diff --git a/tools/ds-identify b/tools/ds-identify index c93d4a77..071cdc0c 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -1062,6 +1062,10 @@ dscheck_OpenStack() { return ${DS_FOUND} fi + if dmi_chassis_asset_tag_matches "SAP CCloud VM"; then + return ${DS_FOUND} + fi + # LP: #1669875 : allow identification of OpenStack by asset tag if dmi_chassis_asset_tag_matches "$nova"; then return ${DS_FOUND} -- cgit v1.2.3 From ed350acb7a941ef16b2f9e19b223b58901e6b431 Mon Sep 17 00:00:00 2001 From: Adam Dobrawy Date: Tue, 31 Mar 2020 19:37:12 +0200 Subject: rbxcloud: gracefully handle arping errors (#262) --- cloudinit/sources/DataSourceRbxCloud.py | 17 +++++++++++----- tests/unittests/test_datasource/test_rbx.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index c3cd5c79..084cb7d5 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -55,11 +55,18 @@ def gratuitous_arp(items, distro): if distro.name in ['fedora', 'centos', 'rhel']: source_param = '-s' for item in items: - _sub_arp([ - '-c', '2', - source_param, item['source'], - item['destination'] - ]) + try: + _sub_arp([ + '-c', '2', + source_param, item['source'], + item['destination'] + ]) + except util.ProcessExecutionError as error: + # warning, because the system is able to function properly + # despite no success - some ARP table may be waiting for + # expiration, but the system may continue + LOG.warning('Failed to arping from "%s" to "%s": %s', + item['source'], item['destination'], error) def get_md(): diff --git a/tests/unittests/test_datasource/test_rbx.py b/tests/unittests/test_datasource/test_rbx.py index aabf1f18..553af62e 100644 --- a/tests/unittests/test_datasource/test_rbx.py +++ b/tests/unittests/test_datasource/test_rbx.py @@ -4,6 +4,7 @@ from cloudinit import helpers from cloudinit import distros from cloudinit.sources import DataSourceRbxCloud as ds from cloudinit.tests.helpers import mock, CiTestCase, populate_dir +from cloudinit import util DS_PATH = "cloudinit.sources.DataSourceRbxCloud" @@ -199,6 +200,35 @@ class TestRbxDataSource(CiTestCase): m_subp.call_args_list ) + @mock.patch( + DS_PATH + '.util.subp', + side_effect=util.ProcessExecutionError() + ) + def test_continue_on_arping_error(self, m_subp): + """Continue when command error""" + items = [ + { + 'destination': '172.17.0.2', + 'source': '172.16.6.104' + }, + { + 'destination': '172.17.0.2', + 'source': '172.16.6.104', + }, + ] + ds.gratuitous_arp(items, self._fetch_distro('ubuntu')) + self.assertEqual([ + mock.call([ + 'arping', '-c', '2', '-S', + '172.16.6.104', '172.17.0.2' + ]), + mock.call([ + 'arping', '-c', '2', '-S', + '172.16.6.104', '172.17.0.2' + ]) + ], m_subp.call_args_list + ) + def populate_cloud_metadata(path, data): populate_dir(path, {'cloud.json': json.dumps(data)}) -- cgit v1.2.3 From d7cad8b61a3b5929a65202e0964aa9b4624e06c4 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2020 14:50:37 -0400 Subject: tests: add missing mocks for get_interfaces_by_mac (#326) We currently have a test system where get_interfaces_by_mac raises an exception, which is causing these tests to fail as they aren't mocking get_interfaces_by_mac out. LP: #1873910 --- cloudinit/sources/tests/test_oracle.py | 2 ++ tests/unittests/test_datasource/test_opennebula.py | 1 + 2 files changed, 3 insertions(+) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py index 4e135df9..2265327b 100644 --- a/cloudinit/sources/tests/test_oracle.py +++ b/cloudinit/sources/tests/test_oracle.py @@ -186,6 +186,7 @@ class TestDataSourceOracle(test_helpers.CiTestCase): self.assertEqual(self.my_md['uuid'], ds.get_instance_id()) self.assertEqual(my_userdata, ds.userdata_raw) + @mock.patch(DS_PATH + ".get_interfaces_by_mac", mock.Mock(return_value={})) @mock.patch(DS_PATH + "._add_network_config_from_opc_imds", side_effect=lambda network_config: network_config) @mock.patch(DS_PATH + ".cmdline.read_initramfs_config") @@ -207,6 +208,7 @@ class TestDataSourceOracle(test_helpers.CiTestCase): self.assertEqual([mock.call()], m_initramfs_config.call_args_list) self.assertFalse(distro.generate_fallback_config.called) + @mock.patch(DS_PATH + ".get_interfaces_by_mac", mock.Mock(return_value={})) @mock.patch(DS_PATH + "._add_network_config_from_opc_imds", side_effect=lambda network_config: network_config) @mock.patch(DS_PATH + ".cmdline.read_initramfs_config") diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index bb399f6d..de896a9e 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -355,6 +355,7 @@ class TestOpenNebulaDataSource(CiTestCase): util.find_devs_with = orig_find_devs_with +@mock.patch(DS_PATH + '.net.get_interfaces_by_mac', mock.Mock(return_value={})) class TestOpenNebulaNetwork(unittest.TestCase): system_nics = ('eth0', 'ens3') -- cgit v1.2.3 From 38a7e6e9756fdab31264c0d6e93d20432ed111ac Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 24 Apr 2020 09:26:51 -0400 Subject: cloudinit: drop dependencies on unittest2 and contextlib2 (#322) These libraries provide backports of Python 3's stdlib components to Python 2. As we only support Python 3, we can simply use the stdlib now. This pull request does the following: * removes some unneeded compatibility code for the old spelling of `assertRaisesRegex` * replaces invocations of the Python 2-only `assertItemsEqual` with its new name, `assertCountEqual` * replaces all usage of `unittest2` with `unittest` * replaces all usage of `contextlib2` with `contextlib` * drops `unittest2` and `contextlib2` from requirements files and tox.ini It also rewrites some `test_azure` helpers to use bare asserts. We were seeing a strange error in xenial builds of this branch which appear to be stemming from the AssertionError that pytest produces being _different_ from the standard AssertionError. This means that the modified helpers weren't behaving correctly, because they weren't catching AssertionErrors as one would expect. (I believe this is related, in some way, to https://github.com/pytest-dev/pytest/issues/645, but the only version of pytest where we're affected is so far in the past that it's not worth pursuing it any further as we have a workaround.) --- cloudinit/config/tests/test_users_groups.py | 10 ++++---- cloudinit/net/tests/test_dhcp.py | 10 ++++---- cloudinit/net/tests/test_init.py | 2 +- cloudinit/sources/tests/test_init.py | 10 ++++---- cloudinit/tests/helpers.py | 28 ++++++---------------- integration-requirements.txt | 1 - packages/pkg-deps.json | 3 --- test-requirements.txt | 2 -- tests/cloud_tests/testcases/__init__.py | 8 +++---- tests/cloud_tests/testcases/base.py | 12 +++++----- tests/cloud_tests/testcases/modules/ntp_chrony.py | 4 ++-- tests/cloud_tests/verify.py | 4 ++-- tests/unittests/test_builtin_handlers.py | 6 ++--- tests/unittests/test_datasource/test_azure.py | 8 +++---- .../unittests/test_datasource/test_azure_helper.py | 6 ++--- tests/unittests/test_datasource/test_smartos.py | 12 +++++----- .../test_handler/test_handler_ca_certs.py | 6 +---- tests/unittests/test_handler/test_handler_chef.py | 8 +++---- .../test_handler/test_handler_growpart.py | 6 +---- tests/unittests/test_handler/test_schema.py | 4 ++-- tox.ini | 2 -- 21 files changed, 61 insertions(+), 91 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/config/tests/test_users_groups.py b/cloudinit/config/tests/test_users_groups.py index f620b597..df89ddb3 100644 --- a/cloudinit/config/tests/test_users_groups.py +++ b/cloudinit/config/tests/test_users_groups.py @@ -39,7 +39,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -65,7 +65,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='freebsd', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_fbsd_user.call_args_list, [mock.call('freebsd', groups='wheel', lock_passwd=True, shell='/bin/tcsh'), @@ -86,7 +86,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -107,7 +107,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -146,7 +146,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index c3fa1e04..bc7bef45 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -62,7 +62,7 @@ class TestParseDHCPLeasesFile(CiTestCase): {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) class TestDHCPRFC3442(CiTestCase): @@ -88,7 +88,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) def test_parse_lease_finds_classless_static_routes(self): """ @@ -114,7 +114,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -324,7 +324,7 @@ class TestDHCPDiscoveryClean(CiTestCase): """) write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -389,7 +389,7 @@ class TestDHCPDiscoveryClean(CiTestCase): write_file(pid_file, "%d\n" % my_pid) m_getppid.return_value = 1 # Indicate that dhclient has daemonized - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 32e70b4c..835ed807 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -397,7 +397,7 @@ class TestGetDeviceList(CiTestCase): """get_devicelist returns a directory listing for SYS_CLASS_NET.""" write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') - self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist()) class TestGetInterfaceMAC(CiTestCase): diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b3648e9..5b6f1b3f 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -350,7 +350,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) sys_info = { @@ -401,7 +401,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}} } - self.assertItemsEqual(expected, redacted) + self.assertCountEqual(expected, redacted) file_stat = os.stat(json_file) self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) @@ -426,7 +426,7 @@ class TestDataSource(CiTestCase): "x86_64"], "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) with mock.patch("cloudinit.util.system_info", return_value=sys_info): @@ -476,7 +476,7 @@ class TestDataSource(CiTestCase): 'security-credentials': {'cred1': 'sekret', 'cred2': 'othersekret'}}}} } - self.assertItemsEqual(expected, util.load_json(content)) + self.assertCountEqual(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) self.assertEqual(expected, util.load_json(content)) @@ -542,7 +542,7 @@ class TestDataSource(CiTestCase): json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) instance_json = util.load_json(content) - self.assertItemsEqual( + self.assertCountEqual( ['ds/meta_data/key2/key2.1'], instance_json['base64_encoded_keys']) self.assertEqual( diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f4db5827..477e14c2 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -13,15 +13,10 @@ import string import sys import tempfile import time +import unittest +from contextlib import ExitStack, contextmanager from unittest import mock - -import unittest2 -from unittest2.util import strclass - -try: - from contextlib import ExitStack, contextmanager -except ImportError: - from contextlib2 import ExitStack, contextmanager +from unittest.util import strclass from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) @@ -35,8 +30,8 @@ from cloudinit import util _real_subp = util.subp # Used for skipping tests -SkipTest = unittest2.SkipTest -skipIf = unittest2.skipIf +SkipTest = unittest.SkipTest +skipIf = unittest.skipIf # Makes the old path start @@ -73,7 +68,7 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): def reset_global_state(self): """Reset any global state to its original settings. @@ -372,7 +367,7 @@ class HttprettyTestCase(CiTestCase): super(HttprettyTestCase, self).tearDown() -class SchemaTestCaseMixin(unittest2.TestCase): +class SchemaTestCaseMixin(unittest.TestCase): def assertSchemaValid(self, cfg, msg="Valid Schema failed validation."): """Assert the config is valid per self.schema. @@ -504,13 +499,4 @@ if not hasattr(mock.Mock, 'assert_not_called'): raise AssertionError(msg) mock.Mock.assert_not_called = __mock_assert_not_called - -# older unittest2.TestCase (centos6) have only the now-deprecated -# assertRaisesRegexp. Simple assignment makes pylint complain, about -# users of assertRaisesRegex so we use getattr to trick it. -# https://github.com/PyCQA/pylint/issues/1946 -if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): - unittest2.TestCase.assertRaisesRegex = ( - getattr(unittest2.TestCase, 'assertRaisesRegexp')) - # vi: ts=4 expandtab diff --git a/integration-requirements.txt b/integration-requirements.txt index 897d6110..44e45c1b 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,7 +5,6 @@ # the packages/pkg-deps.json file as well. # -unittest2 # ec2 backend boto3==1.5.9 diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index d09ec33f..f02e8348 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -10,9 +10,6 @@ "2" : "python-yaml", "3" : "python3-yaml" }, - "contextlib2" : { - "2" : "python-contextlib2" - }, "pyserial" : { "2" : "python-serial", "3" : "python3-serial" diff --git a/test-requirements.txt b/test-requirements.txt index 057115b6..0a6a04d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,7 @@ # Needed generally in tests httpretty>=0.7.1 pytest -unittest2 pytest-cov # Only really needed on older versions of python -contextlib2 setuptools diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index 6bb39f77..e8c371ca 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,7 +4,7 @@ import importlib import inspect -import unittest2 +import unittest from cloudinit.util import read_conf @@ -48,7 +48,7 @@ def get_test_class(test_name, test_data, test_conf): def __str__(self): return "%s (%s)" % (self._testMethodName, - unittest2.util.strclass(self._realclass)) + unittest.util.strclass(self._realclass)) @classmethod def setUpClass(cls): @@ -62,9 +62,9 @@ def get_suite(test_name, data, conf): @return_value: a test suite """ - suite = unittest2.TestSuite() + suite = unittest.TestSuite() suite.addTest( - unittest2.defaultTestLoader.loadTestsFromTestCase( + unittest.defaultTestLoader.loadTestsFromTestCase( get_test_class(test_name, data, conf))) return suite diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index b0dfc144..7b67f54e 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -5,15 +5,15 @@ import crypt import json import re -import unittest2 +import unittest from cloudinit import util as c_util -SkipTest = unittest2.SkipTest +SkipTest = unittest.SkipTest -class CloudTestCase(unittest2.TestCase): +class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" # data gets populated in get_suite.setUpClass @@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys']) + self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -237,7 +237,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys'])) + self.assertCountEqual([], sorted(instance_data['base64_encoded_keys'])) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('lxd', v1_data['platform']) self.assertEqual( @@ -291,7 +291,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], instance_data['base64_encoded_keys']) + self.assertCountEqual([], instance_data['base64_encoded_keys']) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('nocloud', v1_data['platform']) subplatform = v1_data['subplatform'] diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 0f4c3d08..7d341773 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -import unittest2 +import unittest from tests.cloud_tests.testcases import base @@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase): """Skip this suite of tests on lxd and artful or older.""" if self.platform == 'lxd': if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: - raise unittest2.SkipTest( + raise unittest.SkipTest( 'No support for chrony on containers <= artful.' ' LP: #1589780') return super(TestNtpChrony, self).setUp() diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 7018f4d5..0295af40 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest2 +import unittest from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b92ffc79..9045e743 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -109,7 +109,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -120,7 +120,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -302,7 +302,7 @@ class TestConvertJinjaInstanceData(CiTestCase): expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) converted_data = convert_jinja_instance_data(data=data) - self.assertItemsEqual( + self.assertCountEqual( ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) self.assertEqual( expected_data, diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2c6aa44d..2f8561cb 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -528,14 +528,14 @@ scbus-1 on xpt0 bus 0 def tags_exists(x, y): for tag in x.keys(): - self.assertIn(tag, y) + assert tag in y for tag in y.keys(): - self.assertIn(tag, x) + assert tag in x def tags_equal(x, y): for x_val in x.values(): y_val = y.get(x_val.tag) - self.assertEqual(x_val.text, y_val.text) + assert x_val.text == y_val.text old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -649,7 +649,7 @@ scbus-1 on xpt0 bus 0 crawled_metadata = dsrc.crawl_metadata() - self.assertItemsEqual( + self.assertCountEqual( crawled_metadata.keys(), ['cfg', 'files', 'metadata', 'userdata_raw']) self.assertEqual(crawled_metadata['cfg'], expected_cfg) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 007df09f..6344b0bb 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import unittest2 +import unittest from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper @@ -332,7 +332,7 @@ class TestOpenSSLManagerActions(CiTestCase): path = 'tests/data/azure' return os.path.join(path, name) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") def test_pubkey_extract(self): cert = load_file(self._data_file('pubkey_extract_cert')) good_key = load_file(self._data_file('pubkey_extract_ssh_key')) @@ -344,7 +344,7 @@ class TestOpenSSLManagerActions(CiTestCase): fingerprint = sslmgr._get_fingerprint_from_cert(cert) self.assertEqual(good_fingerprint, fingerprint) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") @mock.patch.object(azure_helper.OpenSSLManager, '_decrypt_certs_from_xml') def test_parse_certificates(self, mock_decrypt_certs): """Azure control plane puts private keys as well as certificates diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 62084de5..0f0b42bb 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,7 +22,7 @@ import os.path import re import signal import stat -import unittest2 +import unittest import uuid from cloudinit import serial @@ -1095,11 +1095,11 @@ class TestNetworkConversion(CiTestCase): self.assertEqual(expected, found) -@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, - "Only supported on KVM and bhyve guests under SmartOS") -@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), - "Requires write access to " + SERIAL_DEVICE) -@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available") +@unittest.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, + "Only supported on KVM and bhyve guests under SmartOS") +@unittest.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), + "Requires write access to " + SERIAL_DEVICE) +@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available") class TestSerialConcurrency(CiTestCase): """ This class tests locking on an actual serial port, and as such can only diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 5b4105dd..286ef771 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -11,13 +11,9 @@ import logging import shutil import tempfile import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - class TestNoConfig(unittest.TestCase): def setUp(self): diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 2dab3a54..513c18b5 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -65,18 +65,18 @@ class TestInstallChefOmnibus(HttprettyTestCase): cc_chef.install_chef_from_omnibus() expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) cc_chef.install_chef_from_omnibus(retries=10) expected_kwargs = {'retries': 10, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) expected_subp_kwargs = { 'args': ['-v', '2.0'], 'basename': 'chef-omnibus-install', 'blob': m_rdurl.return_value.contents, 'capture': False } - self.assertItemsEqual( + self.assertCountEqual( expected_subp_kwargs, m_subp_blob.call_args_list[0][1]) @@ -97,7 +97,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): 'blob': response, 'capture': False } - self.assertItemsEqual(expected_kwargs, called_kwargs) + self.assertCountEqual(expected_kwargs, called_kwargs) class TestChef(FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 43b53745..501bcca5 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -11,13 +11,9 @@ import logging import os import re import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - # growpart: # mode: auto # off, on, auto, 'growpart' # devices: ['root'] diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index abde02d3..98628120 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -20,7 +20,7 @@ class GetSchemaTest(CiTestCase): def test_get_schema_coalesces_known_schema(self): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() - self.assertItemsEqual( + self.assertCountEqual( [ 'cc_bootcmd', 'cc_ntp', @@ -38,7 +38,7 @@ class GetSchemaTest(CiTestCase): schema['$schema']) # FULL_SCHEMA is updated by the get_schema call from cloudinit.config.schema import FULL_SCHEMA - self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) + self.assertCountEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) def test_get_schema_returns_global_when_set(self): """When FULL_SCHEMA global is already set, get_schema returns it.""" diff --git a/tox.ini b/tox.ini index 416ddcb8..906519c4 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,6 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 - unittest2==1.1.0 - contextlib2==0.5.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3 From 72f6eb0339f7fcf7b8b02be2e86e5f7477cf731c Mon Sep 17 00:00:00 2001 From: Gonéri Le Bouder Date: Fri, 24 Apr 2020 15:42:36 -0400 Subject: BSD: find_devs_with_ refactoring (#298) Refactoring of the `find_devs_with_*bsd()` methods: - centralize everything in `util.py` - add test coverage --- cloudinit/sources/DataSourceNoCloud.py | 25 ++------- cloudinit/util.py | 52 +++++++++++++---- tests/unittests/test_datasource/test_nocloud.py | 17 +++++- tests/unittests/test_util.py | 74 +++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 32 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 71d5fc2e..72d86c41 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -36,27 +36,14 @@ class DataSourceNoCloud(sources.DataSource): return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def _get_devices(self, label): - if util.is_FreeBSD(): - devlist = [ - p for p in ['/dev/msdosfs/' + label, '/dev/iso9660/' + label] - if os.path.exists(p)] - elif util.is_NetBSD(): - out, _err = util.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) - devlist = [] - for dev in out.split(): - mscdlabel_out, _ = util.subp(['mscdlabel', dev], rcs=[0, 1]) - if ('label "%s"' % label) in mscdlabel_out: - devlist.append('/dev/' + dev) - devlist.append('/dev/' + dev + 'a') # NetBSD 7 - else: - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) - label_list = util.find_devs_with("LABEL=%s" % label.upper()) - label_list.extend(util.find_devs_with("LABEL=%s" % label.lower())) + label_list = util.find_devs_with("LABEL=%s" % label.upper()) + label_list.extend(util.find_devs_with("LABEL=%s" % label.lower())) - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) return devlist def _get_data(self): diff --git a/cloudinit/util.py b/cloudinit/util.py index 818904b1..4cae7ec8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1245,19 +1245,44 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) +def find_devs_with_freebsd(criteria=None, oformat='device', + tag=None, no_cache=False, path=None): + if not criteria: + return glob.glob("/dev/msdosfs/*") + glob.glob("/dev/iso9660/*") + if criteria.startswith("LABEL="): + label = criteria.lstrip("LABEL=") + devlist = [ + p for p in ['/dev/msdosfs/' + label, '/dev/iso9660/' + label] + if os.path.exists(p)] + elif criteria == "TYPE=vfat": + devlist = glob.glob("/dev/msdosfs/*") + elif criteria == "TYPE=iso9660": + devlist = glob.glob("/dev/iso9660/*") + return devlist + + def find_devs_with_netbsd(criteria=None, oformat='device', tag=None, no_cache=False, path=None): - if not path: - path = "/dev/cd0" - cmd = ["mscdlabel", path] - out, _ = subp(cmd, capture=True, decode="replace", rcs=[0, 1]) - result = out.split() - if result and len(result) > 2: - if criteria == "TYPE=iso9660" and "ISO" in result: - return [path] - if criteria == "LABEL=CONFIG-2" and '"config-2"' in result: - return [path] - return [] + devlist = [] + label = None + _type = None + if criteria: + if criteria.startswith("LABEL="): + label = criteria.lstrip("LABEL=") + if criteria.startswith("TYPE="): + _type = criteria.lstrip("TYPE=") + out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + for dev in out.split(): + if label or _type: + mscdlabel_out, _ = subp(['mscdlabel', dev], rcs=[0, 1]) + if label and not ('label "%s"' % label) in mscdlabel_out: + continue + if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: + continue + if _type == "vfat" and "ISO filesystem" in mscdlabel_out: + continue + devlist.append('/dev/' + dev) + return devlist def find_devs_with_openbsd(criteria=None, oformat='device', @@ -1290,7 +1315,10 @@ def find_devs_with(criteria=None, oformat='device', LABEL=