From 45289a00bf8c043c5783c527c4ea720e67e0524b Mon Sep 17 00:00:00 2001 From: Tatiana Kholkina Date: Thu, 1 Feb 2018 18:08:15 +0300 Subject: Fix ssh keys validation in ssh_util This fixes a bug where invalid keys would sneak into authorized_keys. --- tests/unittests/test_sshutil.py | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'tests') diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 2a8e6abe..4c62c8be 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -126,6 +126,48 @@ class TestAuthKeyLineParser(test_helpers.TestCase): self.assertFalse(key.valid()) +class TestUpdateAuthorizedKeys(test_helpers.TestCase): + + def test_new_keys_replace(self): + """new entries with the same base64 should replace old.""" + orig_entries = [ + ' '.join(('rsa', VALID_CONTENT['rsa'], 'orig_comment1')), + ' '.join(('dsa', VALID_CONTENT['dsa'], 'orig_comment2'))] + + new_entries = [ + ' '.join(('rsa', VALID_CONTENT['rsa'], 'new_comment1')), ] + + expected = '\n'.join([new_entries[0], orig_entries[1]]) + '\n' + + parser = ssh_util.AuthKeyLineParser() + found = ssh_util.update_authorized_keys( + [parser.parse(p) for p in orig_entries], + [parser.parse(p) for p in new_entries]) + + self.assertEqual(expected, found) + + def test_new_invalid_keys_are_ignored(self): + """new entries that are invalid should be skipped.""" + orig_entries = [ + ' '.join(('rsa', VALID_CONTENT['rsa'], 'orig_comment1')), + ' '.join(('dsa', VALID_CONTENT['dsa'], 'orig_comment2'))] + + new_entries = [ + ' '.join(('rsa', VALID_CONTENT['rsa'], 'new_comment1')), + 'xxx-invalid-thing1', + 'xxx-invalid-blob2' + ] + + expected = '\n'.join([new_entries[0], orig_entries[1]]) + '\n' + + parser = ssh_util.AuthKeyLineParser() + found = ssh_util.update_authorized_keys( + [parser.parse(p) for p in orig_entries], + [parser.parse(p) for p in new_entries]) + + self.assertEqual(expected, found) + + class TestParseSSHConfig(test_helpers.TestCase): def setUp(self): -- cgit v1.2.3 From 644048e31a9509390871a6a5ab49b92a5e6c3b87 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 9 Feb 2018 09:53:42 -0700 Subject: EC2: Fix get_instance_id called against cached datasource pickle. Fix an issue in EC2 where the datasource.identity had not been initialized before being used when restoring datasource from pickle. This is exposed in upgrade and reboot path. LP: #1748354 --- cloudinit/sources/DataSourceEc2.py | 6 +++ tests/unittests/test_datasource/test_ec2.py | 59 ++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index e14553b3..21e9ef84 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -147,6 +147,12 @@ class DataSourceEc2(sources.DataSource): def get_instance_id(self): if self.cloud_platform == Platforms.AWS: # Prefer the ID from the instance identity document, but fall back + if not getattr(self, 'identity', None): + # If re-using cached datasource, it's get_data run didn't + # setup self.identity. So we need to do that now. + api_version = self.get_metadata_api_version() + self.identity = ec2.get_instance_identity( + api_version, self.metadata_address).get('document', {}) return self.identity.get( 'instanceId', self.metadata['instance-id']) else: diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 0f7267bb..dff8b1ec 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -2,6 +2,7 @@ import copy import httpretty +import json import mock from cloudinit import helpers @@ -9,6 +10,29 @@ from cloudinit.sources import DataSourceEc2 as ec2 from cloudinit.tests import helpers as test_helpers +DYNAMIC_METADATA = { + "instance-identity": { + "document": json.dumps({ + "devpayProductCodes": None, + "marketplaceProductCodes": ["1abc2defghijklm3nopqrs4tu"], + "availabilityZone": "us-west-2b", + "privateIp": "10.158.112.84", + "version": "2017-09-30", + "instanceId": "my-identity-id", + "billingProducts": None, + "instanceType": "t2.micro", + "accountId": "123456789012", + "imageId": "ami-5fb8c835", + "pendingTime": "2016-11-19T16:32:11Z", + "architecture": "x86_64", + "kernelId": None, + "ramdiskId": None, + "region": "us-west-2" + }) + } +} + + # collected from api version 2016-09-02/ with # python3 -c 'import json # from cloudinit.ec2_utils import get_instance_metadata as gm @@ -85,7 +109,7 @@ DEFAULT_METADATA = { "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]}, "reservation-id": "r-01efbc9996bac1bd6", "security-groups": "my-wide-open", - "services": {"domain": "amazonaws.com", "partition": "aws"} + "services": {"domain": "amazonaws.com", "partition": "aws"}, } @@ -340,6 +364,39 @@ class TestEc2(test_helpers.HttprettyTestCase): 'type': 'physical'}]} self.assertEqual(expected, ds.network_config) + @httpretty.activate + def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self): + """get_instance-id gets DataSourceEc2Local.identity if not present. + + This handles an upgrade case where the old pickled datasource didn't + set up self.identity, but 'systemctl cloud-init init' runs + get_instance_id which traces on missing self.identity. lp:1748354. + """ + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + # Mock 404s on all versions except latest + all_versions = ( + [ds.min_metadata_version] + ds.extended_metadata_versions) + for ver in all_versions[:-1]: + register_mock_metaserver( + 'http://169.254.169.254/{0}/meta-data/instance-id'.format(ver), + None) + ds.metadata_address = 'http://169.254.169.254' + register_mock_metaserver( + '{0}/{1}/meta-data/'.format(ds.metadata_address, all_versions[-1]), + DEFAULT_METADATA) + # Register dynamic/instance-identity document which we now read. + register_mock_metaserver( + '{0}/{1}/dynamic/'.format(ds.metadata_address, all_versions[-1]), + DYNAMIC_METADATA) + ds._cloud_platform = ec2.Platforms.AWS + # Setup cached metadata on the Datasource + ds.metadata = DEFAULT_METADATA + self.assertEqual('my-identity-id', ds.get_instance_id()) + @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): -- cgit v1.2.3 From f576b2a24b8014e91087933d19a7a0d396787c30 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 26 Jan 2018 16:00:10 -0500 Subject: tests: add support for logs with lxd from snap and future lxd 3. This puts in place detection for if 'show-log' will work with lxc client, and uses that if present. The 'lxc console --show-log' is not expected to work until lxd/liblxc3.0. That should come in a few months. The hope is that when that function arrives, this code will move over to using it. For other scenarios (all current lxd installs) this will now support getting logs from a snap installed lxd or a package installed lxd via the old 'lxc.console.logfile'. If installed from snap, a platform error will be raised until the user does: sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles LP: #1745663 --- tests/cloud_tests/collect.py | 5 +- tests/cloud_tests/platforms/lxd/instance.py | 132 ++++++++++++++++++++++------ 2 files changed, 109 insertions(+), 28 deletions(-) (limited to 'tests') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 5ea88e50..d4f9135b 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -44,8 +44,9 @@ def collect_console(instance, base_dir): LOG.debug('getting console log for %s to %s', instance, logfile) try: data = instance.console_log() - except NotImplementedError: - data = b'instance.console_log: not implemented' + except NotImplementedError as e: + # args[0] is hacky, but thats all I see to get at the message. + data = b'NotImplementedError:' + e.args[0].encode() with open(logfile, "wb") as fp: fp.write(data) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index d2d2a1fd..0488da57 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -6,7 +6,9 @@ import os import shutil from tempfile import mkdtemp -from cloudinit.util import subp, ProcessExecutionError +from cloudinit.util import load_yaml, subp, ProcessExecutionError, which +from tests.cloud_tests import LOG +from tests.cloud_tests.util import PlatformError from ..instances import Instance @@ -15,6 +17,8 @@ class LXDInstance(Instance): """LXD container backed instance.""" platform_name = "lxd" + _console_log_method = None + _console_log_file = None def __init__(self, platform, name, properties, config, features, pylxd_container): @@ -30,8 +34,8 @@ class LXDInstance(Instance): super(LXDInstance, self).__init__( platform, name, properties, config, features) self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) - self._setup_console_log() self.name = name + self._setup_console_log() @property def pylxd_container(self): @@ -39,21 +43,6 @@ class LXDInstance(Instance): self._pylxd_container.sync() return self._pylxd_container - def _setup_console_log(self): - logf = os.path.join(self.tmpd, "console.log") - - # doing this ensures we can read it. Otherwise it ends up root:root. - with open(logf, "w") as fp: - fp.write("# %s\n" % self.name) - - cfg = "lxc.console.logfile=%s" % logf - orig = self._pylxd_container.config.get('raw.lxc', "") - if orig: - orig += "\n" - self._pylxd_container.config['raw.lxc'] = orig + cfg - self._pylxd_container.save() - self._console_log_file = logf - def _execute(self, command, stdin=None, env=None): if env is None: env = {} @@ -97,19 +86,80 @@ class LXDInstance(Instance): """ self.pylxd_container.files.put(remote_path, data) + @property + def console_log_method(self): + if self._console_log_method is not None: + return self._console_log_method + + client = which('lxc') + if not client: + raise PlatformError("No 'lxc' client.") + + elif _has_proper_console_support(): + self._console_log_method = 'show-log' + elif client.startswith("/snap"): + self._console_log_method = 'logfile-snap' + else: + self._console_log_method = 'logfile-tmp' + + LOG.debug("Set console log method to %s", self._console_log_method) + return self._console_log_method + + def _setup_console_log(self): + method = self.console_log_method + if not method.startswith("logfile-"): + return + + if method == "logfile-snap": + log_dir = "/var/snap/lxd/common/consoles" + if not os.path.exists(log_dir): + raise PlatformError( + "Unable to log with snap lxc. Please run:\n" + " sudo mkdir --mode=1777 -p %s" % log_dir) + elif method == "logfile-tmp": + log_dir = "/tmp" + else: + raise PlatformError( + "Unexpected value for console method: %s" % method) + + # doing this ensures we can read it. Otherwise it ends up root:root. + log_file = os.path.join(log_dir, self.name) + with open(log_file, "w") as fp: + fp.write("# %s\n" % self.name) + + cfg = "lxc.console.logfile=%s" % log_file + orig = self._pylxd_container.config.get('raw.lxc', "") + if orig: + orig += "\n" + self._pylxd_container.config['raw.lxc'] = orig + cfg + self._pylxd_container.save() + self._console_log_file = log_file + def console_log(self): """Console log. - @return_value: bytes of this instance’s console + @return_value: bytes of this instance's console """ - if not os.path.exists(self._console_log_file): - raise NotImplementedError( - "Console log '%s' does not exist. If this is a remote " - "lxc, then this is really NotImplementedError. If it is " - "A local lxc, then this is a RuntimeError." - "https://github.com/lxc/lxd/issues/1129") - with open(self._console_log_file, "rb") as fp: - return fp.read() + + if self._console_log_file: + if not os.path.exists(self._console_log_file): + raise NotImplementedError( + "Console log '%s' does not exist. If this is a remote " + "lxc, then this is really NotImplementedError. If it is " + "A local lxc, then this is a RuntimeError." + "https://github.com/lxc/lxd/issues/1129") + with open(self._console_log_file, "rb") as fp: + return fp.read() + + try: + stdout, stderr = subp( + ['lxc', 'console', '--show-log', self.name], decode=False) + return stdout + except ProcessExecutionError as e: + raise PlatformError( + "console log", + "Console log failed [%d]: stdout=%s stderr=%s" % ( + e.exit_code, e.stdout, e.stderr)) def reboot(self, wait=True): """Reboot instance.""" @@ -146,7 +196,37 @@ class LXDInstance(Instance): if self.platform.container_exists(self.name): raise OSError('container {} was not properly removed' .format(self.name)) + if self._console_log_file and os.path.exists(self._console_log_file): + os.unlink(self._console_log_file) shutil.rmtree(self.tmpd) super(LXDInstance, self).destroy() + +def _has_proper_console_support(): + stdout, _ = subp(['lxc', 'info']) + info = load_yaml(stdout) + reason = None + if 'console' not in info.get('api_extensions', []): + reason = "LXD server does not support console api extension" + else: + dver = info.get('environment', {}).get('driver_version', "") + if dver.startswith("2.") or dver.startwith("1."): + reason = "LXD Driver version not 3.x+ (%s)" % dver + else: + try: + stdout, stderr = subp(['lxc', 'console', '--help'], + decode=False) + if not (b'console' in stdout and b'log' in stdout): + reason = "no '--log' in lxc console --help" + except ProcessExecutionError as e: + reason = "no 'console' command in lxc client" + + if reason: + LOG.debug("no console-support: %s", reason) + return False + else: + LOG.debug("console-support looks good") + return True + + # vi: ts=4 expandtab -- cgit v1.2.3 From 754f54037aca0f604b8b57ab71b30dad5e5066cf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Feb 2018 13:54:50 -0700 Subject: tests: run nosetests in cloudinit/ directory, fix py26 fallout. When we moved some tests to live under cloudinit/ we inadvertantly failed to change all things that would run nose to include that directory. This changes all the 'nose' invocations to consistently run with tests/unittests and cloudinit/. Also, it works around, more correctly this time, a python2.6-ism with the following code: with assertRaises(SystemExit) as cm: sys.exit(2) --- cloudinit/cmd/tests/test_clean.py | 3 ++- cloudinit/cmd/tests/test_status.py | 3 ++- cloudinit/tests/helpers.py | 20 +++++++++----------- tests/unittests/test_handler/test_schema.py | 12 +++++++----- tools/run-centos | 3 ++- tox.ini | 6 +++--- 6 files changed, 25 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/cloudinit/cmd/tests/test_clean.py b/cloudinit/cmd/tests/test_clean.py index 6713af4f..5a3ec3bf 100644 --- a/cloudinit/cmd/tests/test_clean.py +++ b/cloudinit/cmd/tests/test_clean.py @@ -165,10 +165,11 @@ class TestClean(CiTestCase): wrap_and_call( 'cloudinit.cmd.clean', {'Init': {'side_effect': self.init_class}, + 'sys.exit': {'side_effect': self.sys_exit}, 'sys.argv': {'new': ['clean', '--logs']}}, clean.main) - self.assertRaisesCodeEqual(0, context_manager.exception.code) + self.assertEqual(0, context_manager.exception.code) self.assertFalse( os.path.exists(self.log1), 'Unexpected log {0}'.format(self.log1)) diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py index 4a5a8c06..37a89936 100644 --- a/cloudinit/cmd/tests/test_status.py +++ b/cloudinit/cmd/tests/test_status.py @@ -380,10 +380,11 @@ class TestStatus(CiTestCase): wrap_and_call( 'cloudinit.cmd.status', {'sys.argv': {'new': ['status']}, + 'sys.exit': {'side_effect': self.sys_exit}, '_is_cloudinit_disabled': (False, ''), 'Init': {'side_effect': self.init_class}}, status.main) - self.assertRaisesCodeEqual(0, context_manager.exception.code) + self.assertEqual(0, context_manager.exception.code) self.assertEqual('status: running\n', m_stdout.getvalue()) # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 0080c729..41d9a8ee 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -173,17 +173,15 @@ class CiTestCase(TestCase): dir = self.tmp_dir() return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - def assertRaisesCodeEqual(self, expected, found): - """Handle centos6 having different context manager for assertRaises. - with assertRaises(Exception) as e: - raise Exception("BOO") - - centos6 will have e.exception as an integer. - anything nwere will have it as something with a '.code'""" - if isinstance(found, int): - self.assertEqual(expected, found) - else: - self.assertEqual(expected, found.code) + def sys_exit(self, code): + """Provide a wrapper around sys.exit for python 2.6 + + In 2.6, this code would produce 'cm.exception' with value int(2) + rather than the SystemExit that was raised by sys.exit(2). + with assertRaises(SystemExit) as cm: + sys.exit(2) + """ + raise SystemExit(code) class ResourceUsingTestCase(CiTestCase): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 648573f6..df67a0e0 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -336,11 +336,13 @@ class MainTest(CiTestCase): def test_main_missing_args(self): """Main exits non-zero and reports an error on missing parameters.""" - with mock.patch('sys.argv', ['mycmd']): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual('1', str(context_manager.exception)) + with mock.patch('sys.exit', side_effect=self.sys_exit): + with mock.patch('sys.argv', ['mycmd']): + with mock.patch('sys.stderr', new_callable=StringIO) as \ + m_stderr: + with self.assertRaises(SystemExit) as context_manager: + main() + self.assertEqual(1, context_manager.exception.code) self.assertEqual( 'Expected either --config-file argument or --doc\n', m_stderr.getvalue()) diff --git a/tools/run-centos b/tools/run-centos index 6ac6c11f..cb241ee5 100755 --- a/tools/run-centos +++ b/tools/run-centos @@ -313,7 +313,8 @@ main() { if [ -n "$unittest" ]; then debug 1 "running unit tests." - inside_as_cd "$name" "$user" "$cdir" nosetests tests/unittests || + inside_as_cd "$name" "$user" "$cdir" \ + nosetests tests/unittests cloudinit || { errorrc "nosetests failed."; errors=$(($errors+1)); } fi diff --git a/tox.ini b/tox.ini index bb74853f..1f990af4 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:py26] deps = -r{toxinidir}/test-requirements.txt -commands = nosetests {posargs:tests/unittests} +commands = nosetests {posargs:tests/unittests cloudinit} setenv = LC_ALL = C @@ -83,7 +83,7 @@ deps = [testenv:centos6] basepython = python2.6 -commands = nosetests {posargs:tests/unittests} +commands = nosetests {posargs:tests/unittests cloudinit} deps = # requirements argparse==1.2.1 @@ -98,7 +98,7 @@ deps = [testenv:opensusel42] basepython = python2.7 -commands = nosetests {posargs:tests/unittests} +commands = nosetests {posargs:tests/unittests cloudinit} deps = # requirements argparse==1.3.0 -- cgit v1.2.3 From 42923ffbbf6a84819224417f0e032497af60d0cd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Feb 2018 15:10:25 -0700 Subject: ds-identify: check /writable/system-data/ for nocloud seed. Ubuntu core seeds information to nocloud via a bind-mount of /writable/system-data/var/lib/cloud over /var/lib/cloud. When ds-identify runs as a systemd generator that mount is not guaranteed to have been done. It is guaranteed at cloud-init-local.service time, but not generator time. Images built with 'ubuntu-image --cloud-init=user-data-file' would have cloud-init disabled. The fix here is just to consider the seed dir under /writable/system-data. LP: #1747070 --- tests/unittests/test_ds_identify.py | 24 ++++++++++++++++++++++++ tools/ds-identify | 11 +++++++++++ 2 files changed, 35 insertions(+) (limited to 'tests') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 31cc6223..03942de0 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -359,6 +359,14 @@ class TestDsIdentify(CiTestCase): """NoCloud is found with iso9660 filesystem on non-cdrom disk.""" self._test_ds_found('NoCloud') + def test_nocloud_seed(self): + """Nocloud seed directory.""" + self._test_ds_found('NoCloud-seed') + + def test_nocloud_seed_ubuntu_core_writable(self): + """Nocloud seed directory ubuntu core writable""" + self._test_ds_found('NoCloud-seed-ubuntu-core') + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" @@ -454,6 +462,22 @@ VALID_CFG = { 'dev/vdb': 'pretend iso content for cidata\n', } }, + 'NoCloud-seed': { + 'ds': 'NoCloud', + 'files': { + os.path.join(P_SEED_DIR, 'nocloud', 'user-data'): 'ud\n', + os.path.join(P_SEED_DIR, 'nocloud', 'meta-data'): 'md\n', + } + }, + 'NoCloud-seed-ubuntu-core': { + 'ds': 'NoCloud', + 'files': { + os.path.join('writable/system-data', P_SEED_DIR, + 'nocloud-net', 'user-data'): 'ud\n', + os.path.join('writable/system-data', P_SEED_DIR, + 'nocloud-net', 'meta-data'): 'md\n', + } + }, 'OpenStack': { 'ds': 'OpenStack', 'files': {P_PRODUCT_NAME: 'OpenStack Nova\n'}, diff --git a/tools/ds-identify b/tools/ds-identify index cd268242..5f762438 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -470,6 +470,16 @@ check_seed_dir() { return 0 } +check_writable_seed_dir() { + # ubuntu core bind-mounts /writable/system-data/var/lib/cloud + # over the top of /var/lib/cloud, but the mount might not be done yet. + local wdir="/writable/system-data" + [ -d "${PATH_ROOT}$wdir" ] || return 1 + local sdir="${PATH_ROOT}$wdir${PATH_VAR_LIB_CLOUD#${PATH_ROOT}}" + local PATH_VAR_LIB_CLOUD="$sdir" + check_seed_dir "$@" +} + probe_floppy() { cached "${STATE_FLOPPY_PROBED}" && return "${STATE_FLOPPY_PROBED}" local fpath=/dev/floppy @@ -569,6 +579,7 @@ dscheck_NoCloud() { esac for d in nocloud nocloud-net; do check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} + check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND} done if has_fs_with_label "${fslabel}"; then return ${DS_FOUND} -- cgit v1.2.3 From b7497e807fa12a26d4a12aaf1ee9302a4fd24728 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 21 Feb 2018 16:09:14 -0500 Subject: ds-identify: Fix searching for iso9660 OVF cdroms. This fixes a bug in parsing of 'blkid -o export' output. The result of the bug meant that DI_ISO9660_DEVS did not get set correctly and is_cdrom_ovf would not identify devices in most cases. The tests are improved to demonstrate both multiple iso devices and also a cdrom that doesn't sort "last" in blkid output. The code change is to use DEVNAME as the record separator when parsing blkid -o export rather than relying on being able to read the empty line. LP: #1749980 --- tests/unittests/test_ds_identify.py | 9 ++++++--- tools/ds-identify | 39 ++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 21 deletions(-) (limited to 'tests') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 03942de0..21258347 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -350,8 +350,10 @@ class TestDsIdentify(CiTestCase): "OVFENV", "ovfenv"] for valid_ovf_label in valid_ovf_labels: ovf_cdrom_by_label['mocks'][0]['out'] = blkid_out([ + {'DEVNAME': 'sda1', 'TYPE': 'ext4', 'LABEL': 'rootfs'}, {'DEVNAME': 'sr0', 'TYPE': 'iso9660', - 'LABEL': valid_ovf_label}]) + 'LABEL': valid_ovf_label}, + {'DEVNAME': 'vda1', 'TYPE': 'ntfs', 'LABEL': 'data'}]) self._check_via_dict( ovf_cdrom_by_label, rc=RC_FOUND, dslist=['OVF', DS_NONE]) @@ -513,8 +515,9 @@ VALID_CFG = { 'mocks': [ {'name': 'blkid', 'ret': 0, 'out': blkid_out( - [{'DEVNAME': 'vda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, - {'DEVNAME': 'sr0', 'TYPE': 'iso9660', 'LABEL': ''}]) + [{'DEVNAME': 'sr0', 'TYPE': 'iso9660', 'LABEL': ''}, + {'DEVNAME': 'sr1', 'TYPE': 'iso9660', 'LABEL': 'ignoreme'}, + {'DEVNAME': 'vda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}]), }, MOCK_VIRT_IS_VMWARE, ], diff --git a/tools/ds-identify b/tools/ds-identify index 5f762438..5da51bcc 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -186,7 +186,8 @@ block_dev_with_label() { read_fs_info() { cached "${DI_BLKID_OUTPUT}" && return 0 # do not rely on links in /dev/disk which might not be present yet. - # note that older blkid versions do not report DEVNAME in 'export' output. + # Note that blkid < 2.22 (centos6, trusty) do not output DEVNAME. + # that means that DI_ISO9660_DEVS will not be set. if is_container; then # blkid will in a container, or at least currently in lxd # not provide useful information. @@ -203,21 +204,26 @@ read_fs_info() { DI_ISO9660_DEVS="$UNAVAILABLE:error" return $ret } - IFS="$CR" - set -- $out - IFS="$oifs" - for line in "$@" ""; do + # 'set --' will collapse multiple consecutive entries in IFS for + # whitespace characters (\n, tab, " ") so we cannot rely on getting + # empty lines in "$@" below. + IFS="$CR"; set -- $out; IFS="$oifs" + + for line in "$@"; do case "${line}" in - DEVNAME=*) dev=${line#DEVNAME=};; + DEVNAME=*) + [ -n "$dev" -a "$ftype" = "iso9660" ] && + isodevs="${isodevs} ${dev}=$label" + ftype=""; dev=""; label=""; + dev=${line#DEVNAME=};; LABEL=*) label="${line#LABEL=}"; labels="${labels}${line#LABEL=}${delim}";; TYPE=*) ftype=${line#TYPE=};; - "") if [ "$ftype" = "iso9660" ]; then - isodevs="${isodevs} ${dev}=$label" - fi - ftype=""; devname=""; label=""; esac done + [ -n "$dev" -a "$ftype" = "iso9660" ] && + isodevs="${isodevs} ${dev}=$label" + DI_FS_LABELS="${labels%${delim}}" DI_ISO9660_DEVS="${isodevs# }" } @@ -696,15 +702,12 @@ dscheck_OVF() { # Azure provides ovf. Skip false positive by dis-allowing. is_azure_chassis && return $DS_NOT_FOUND - local isodevs="${DI_ISO9660_DEVS}" - case "$isodevs" in - ""|$UNAVAILABLE:*) return ${DS_NOT_FOUND};; - esac - # DI_ISO9660_DEVS is =label, like /dev/sr0=OVF-TRANSPORT - for tok in $isodevs; do - is_cdrom_ovf "${tok%%=*}" "${tok#*=}" && return $DS_FOUND - done + if [ "${DI_ISO9660_DEVS#${UNAVAILABLE}:}" = "${DI_ISO9660_DEVS}" ]; then + for tok in ${DI_ISO9660_DEVS}; do + is_cdrom_ovf "${tok%%=*}" "${tok#*=}" && return $DS_FOUND + done + fi if ovf_vmware_guest_customization; then return ${DS_FOUND} -- cgit v1.2.3 From 3814d559c3e973238d819721605c7451e852fe63 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 13 Feb 2018 13:00:53 -0800 Subject: OVF: Fix VMware support for 64-bit platforms. On few 64-bit platforms, the open-vm-tools package is installed at /usr/lib64/. The DataSourceOVF is changed to search look there for the 'customization plugin' --- cloudinit/sources/DataSourceOVF.py | 21 ++++++++++++++++----- tests/unittests/test_ds_identify.py | 10 ++++++++++ tools/ds-identify | 3 ++- 3 files changed, 28 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 6e62f984..dc914a72 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -95,11 +95,20 @@ class DataSourceOVF(sources.DataSource): "VMware Customization support") elif not util.get_cfg_option_bool( self.sys_cfg, "disable_vmware_customization", True): - deployPkgPluginPath = search_file("/usr/lib/vmware-tools", - "libdeployPkgPlugin.so") - if not deployPkgPluginPath: - deployPkgPluginPath = search_file("/usr/lib/open-vm-tools", - "libdeployPkgPlugin.so") + + search_paths = ( + "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools", + "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools") + + plugin = "libdeployPkgPlugin.so" + deployPkgPluginPath = None + for path in search_paths: + deployPkgPluginPath = search_file(path, plugin) + if deployPkgPluginPath: + LOG.debug("Found the customization plugin at %s", + deployPkgPluginPath) + break + if deployPkgPluginPath: # When the VM is powered on, the "VMware Tools" daemon # copies the customization specification file to @@ -111,6 +120,8 @@ class DataSourceOVF(sources.DataSource): msg="waiting for configuration file", func=wait_for_imc_cfg_file, args=("cust.cfg", max_wait)) + else: + LOG.debug("Did not find the customization plugin.") if vmwareImcConfigFilePath: LOG.debug("Found VMware Customization Config File at %s", diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 21258347..9be3f964 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -337,6 +337,16 @@ class TestDsIdentify(CiTestCase): """OVF is identified when vmware customization is enabled.""" self._test_ds_found('OVF-vmware-customization') + def test_ovf_on_vmware_iso_found_open_vm_tools_64(self): + """OVF is identified when open-vm-tools installed in /usr/lib64.""" + cust64 = copy.deepcopy(VALID_CFG['OVF-vmware-customization']) + p32 = 'usr/lib/vmware-tools/plugins/vmsvc/libdeployPkgPlugin.so' + open64 = 'usr/lib64/open-vm-tools/plugins/vmsvc/libdeployPkgPlugin.so' + cust64['files'][open64] = cust64['files'][p32] + del cust64['files'][p32] + return self._check_via_dict( + cust64, RC_FOUND, dslist=[cust64.get('ds'), DS_NONE]) + def test_ovf_on_vmware_iso_found_by_cdrom_with_matching_fs_label(self): """OVF is identified by well-known iso9660 labels.""" ovf_cdrom_by_label = copy.deepcopy(VALID_CFG['OVF']) diff --git a/tools/ds-identify b/tools/ds-identify index 5da51bcc..ec368d58 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -650,8 +650,9 @@ ovf_vmware_guest_customization() { # we have to have the plugin to do vmware customization local found="" pkg="" pre="${PATH_ROOT}/usr/lib" + local ppath="plugins/vmsvc/libdeployPkgPlugin.so" for pkg in vmware-tools open-vm-tools; do - if [ -f "$pre/$pkg/plugins/vmsvc/libdeployPkgPlugin.so" ]; then + if [ -f "$pre/$pkg/$ppath" -o -f "${pre}64/$pkg/$ppath" ]; then found="$pkg"; break; fi done -- cgit v1.2.3 From 65f0b263a1af170802215823d6e354f8bdfa5f39 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 4 Dec 2017 16:49:15 -0700 Subject: salt: configure grains in grains file rather than in minion config. While salt grains can be configured in the minion config file, it is usually better to configure it in the /etc/salt/grains file. This allows that to be done. --- cloudinit/config/cc_salt_minion.py | 9 +++++++++ tests/cloud_tests/testcases/modules/salt_minion.py | 5 +++++ tests/cloud_tests/testcases/modules/salt_minion.yaml | 5 +++++ 3 files changed, 19 insertions(+) (limited to 'tests') diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 2b388372..5112a347 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -25,6 +25,9 @@ specified with ``public_key`` and ``private_key`` respectively. salt_minion: conf: master: salt.example.com + grains: + role: + - web public_key: | ------BEGIN PUBLIC KEY------- @@ -65,6 +68,12 @@ def handle(name, cfg, cloud, log, _args): minion_data = util.yaml_dumps(salt_cfg.get('conf')) util.write_file(minion_config, minion_data) + if 'grains' in salt_cfg: + # add grains to /etc/salt/grains + grains_config = os.path.join(config_dir, 'grains') + grains_data = util.yaml_dumps(salt_cfg.get('grains')) + util.write_file(grains_config, grains_data) + # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: if os.path.isdir("/etc/salt/pki/minion"): diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py index c697db2d..f13b48a0 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ b/tests/cloud_tests/testcases/modules/salt_minion.py @@ -26,4 +26,9 @@ class Test(base.CloudTestCase): self.assertIn('', out) self.assertIn('------END PUBLIC KEY-------', out) + def test_grains(self): + """Test master value in config.""" + out = self.get_data_file('grains') + self.assertIn('role: web', out) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml index f20d24f0..ab0e05bb 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -17,6 +17,8 @@ cloud_config: | ------BEGIN PRIVATE KEY------ ------END PRIVATE KEY------- + grains: + role: web collect_scripts: minion: | #!/bin/bash @@ -30,5 +32,8 @@ collect_scripts: minion.pub: | #!/bin/bash cat /etc/salt/pki/minion/minion.pub + grains: | + #!/bin/bash + cat /etc/salt/grains # vi: ts=4 expandtab -- cgit v1.2.3 From 46cb6716c27d4496ce3d2bea7684803f522f277d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 22 Feb 2018 16:18:02 -0500 Subject: subp: Fix subp usage with non-ascii characters when no system locale. If python starts up without a locale set, then its default encoding ends up set as ascii. That is not easily changed with the likes of setlocale. In order to avoid UnicodeDecodeErrors cloud-init will encode to bytes a python3 string or python2 basestring so that the values passed to Popen are already bytes. LP: #1751051 --- cloudinit/util.py | 7 ++++++- tests/unittests/test_util.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/cloudinit/util.py b/cloudinit/util.py index 338fb971..5a919cfe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1865,8 +1865,13 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, if not isinstance(data, bytes): data = data.encode() + # Popen converts entries in the arguments array from non-bytes to bytes. + # When locale is unset it may use ascii for that encoding which can + # cause UnicodeDecodeErrors. (LP: #1751051) + bytes_args = [x if isinstance(x, six.binary_type) else x.encode("utf-8") + for x in args] try: - sp = subprocess.Popen(args, stdout=stdout, + sp = subprocess.Popen(bytes_args, stdout=stdout, stderr=stderr, stdin=stdin, env=env, shell=shell) (out, err) = sp.communicate(data) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 4a92e741..89ae40f5 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -8,7 +8,9 @@ import shutil import stat import tempfile +import json import six +import sys import yaml from cloudinit import importer, util @@ -733,6 +735,38 @@ class TestSubp(helpers.CiTestCase): self.assertEqual("/target/my/path/", util.target_path("/target/", "///my/path/")) + def test_c_lang_can_take_utf8_args(self): + """Independent of system LC_CTYPE, args can contain utf-8 strings. + + When python starts up, its default encoding gets set based on + the value of LC_CTYPE. If no system locale is set, the default + encoding for both python2 and python3 in some paths will end up + being ascii. + + Attempts to use setlocale or patching (or changing) os.environ + in the current environment seem to not be effective. + + This test starts up a python with LC_CTYPE set to C so that + the default encoding will be set to ascii. In such an environment + Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. + """ + python_prog = '\n'.join([ + 'import json, sys', + 'from cloudinit.util import subp', + 'data = sys.stdin.read()', + 'cmd = json.loads(data)', + 'subp(cmd, capture=False)', + '']) + cmd = [BASH, '-c', 'echo -n "$@"', '--', + self.utf8_valid.decode("utf-8")] + python_subp = [sys.executable, '-c', python_prog] + + out, _err = util.subp( + python_subp, update_env={'LC_CTYPE': 'C'}, + data=json.dumps(cmd).encode("utf-8"), + decode=False) + self.assertEqual(self.utf8_valid, out) + class TestEncode(helpers.TestCase): """Test the encoding functions""" -- cgit v1.2.3 From 40e77380e036a24fafe91a63d0cdefada4312348 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 1 Mar 2018 15:39:16 -0700 Subject: GCE: fix reading of user-data that is not base64 encoded. Last set of changes to GCE datasource broke reading of user-data unless the user had base64 encoded their user-data and also set user-data-encoding to 'base64'. This fixes the issue. LP: #1752711 --- cloudinit/sources/DataSourceGCE.py | 15 +++++++-------- tests/unittests/test_datasource/test_gce.py | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2da34a99..bebc9918 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -213,16 +213,15 @@ def read_md(address=None, platform_check=True): if md['availability-zone']: md['availability-zone'] = md['availability-zone'].split('/')[-1] - encoding = instance_data.get('user-data-encoding') - if encoding: + if 'user-data' in instance_data: + # instance_data was json, so values are all utf-8 strings. + ud = instance_data['user-data'].encode("utf-8") + encoding = instance_data.get('user-data-encoding') if encoding == 'base64': - md['user-data'] = b64decode(instance_data.get('user-data')) - else: + ud = b64decode(ud) + elif encoding: LOG.warning('unknown user-data-encoding: %s, ignoring', encoding) - - if 'user-data' in md: - ret['user-data'] = md['user-data'] - del md['user-data'] + ret['user-data'] = ud ret['meta-data'] = md ret['success'] = True diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index f77c2c40..eb3cec42 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -38,11 +38,20 @@ GCE_META_ENCODING = { 'instance/hostname': 'server.project-baz.local', 'instance/zone': 'baz/bang', 'instance/attributes': { - 'user-data': b64encode(b'/bin/echo baz\n').decode('utf-8'), + 'user-data': b64encode(b'#!/bin/echo baz\n').decode('utf-8'), 'user-data-encoding': 'base64', } } +GCE_USER_DATA_TEXT = { + 'instance/id': '12345', + 'instance/hostname': 'server.project-baz.local', + 'instance/zone': 'baz/bang', + 'instance/attributes': { + 'user-data': '#!/bin/sh\necho hi mom\ntouch /run/up-now\n', + } +} + HEADERS = {'Metadata-Flavor': 'Google'} MD_URL_RE = re.compile( r'http://metadata.google.internal/computeMetadata/v1/.*') @@ -135,7 +144,16 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): shostname = GCE_META_PARTIAL.get('instance/hostname').split('.')[0] self.assertEqual(shostname, self.ds.get_hostname()) + def test_userdata_no_encoding(self): + """check that user-data is read.""" + _set_mock_metadata(GCE_USER_DATA_TEXT) + self.ds.get_data() + self.assertEqual( + GCE_USER_DATA_TEXT['instance/attributes']['user-data'].encode(), + self.ds.get_userdata_raw()) + def test_metadata_encoding(self): + """user-data is base64 encoded if user-data-encoding is 'base64'.""" _set_mock_metadata(GCE_META_ENCODING) self.ds.get_data() -- cgit v1.2.3 From 1e2e810f3f7cb6a163a0229ac37037e8c6744d72 Mon Sep 17 00:00:00 2001 From: Dominic Schlegel Date: Thu, 8 Mar 2018 12:13:38 +0100 Subject: Make salt minion module work on FreeBSD. Previously the module was not working under FreeBSD due to a different package name and some different paths. The module now has OS specific default values which can even be customized via corresponding cloud config variables. LP: #1721503 --- cloudinit/config/cc_salt_minion.py | 82 ++++++++++++++++------ config/cloud.cfg.tmpl | 2 +- tests/cloud_tests/testcases/modules/salt_minion.py | 5 ++ .../cloud_tests/testcases/modules/salt_minion.yaml | 5 +- 4 files changed, 69 insertions(+), 25 deletions(-) (limited to 'tests') diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 5112a347..d6a21d72 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -12,7 +12,9 @@ key is present in the config parts, then salt minion will be installed and started. Configuration for salt minion can be specified in the ``conf`` key under ``salt_minion``. Any conf values present there will be assigned in ``/etc/salt/minion``. The public and private keys to use for salt minion can be -specified with ``public_key`` and ``private_key`` respectively. +specified with ``public_key`` and ``private_key`` respectively. Optionally if +you have a custom package name, service name or config directory you can +specify them with ``pkg_name``, ``service_name`` and ``config_dir``. **Internal name:** ``cc_salt_minion`` @@ -23,6 +25,9 @@ specified with ``public_key`` and ``private_key`` respectively. **Config keys**:: salt_minion: + pkg_name: 'salt-minion' + service_name: 'salt-minion' + config_dir: '/etc/salt' conf: master: salt.example.com grains: @@ -42,7 +47,34 @@ import os from cloudinit import util -# Note: see http://saltstack.org/topics/installation/ +# Note: see https://docs.saltstack.com/en/latest/topics/installation/ +# Note: see https://docs.saltstack.com/en/latest/ref/configuration/ + + +class SaltConstants(object): + """ + defines default distribution specific salt variables + """ + def __init__(self, cfg): + + # constants tailored for FreeBSD + if util.is_FreeBSD(): + self.pkg_name = 'py27-salt' + self.srv_name = 'salt_minion' + self.conf_dir = '/usr/local/etc/salt' + # constants for any other OS + else: + self.pkg_name = 'salt-minion' + self.srv_name = 'salt-minion' + self.conf_dir = '/etc/salt' + + # if there are constants given in cloud config use those + self.pkg_name = util.get_cfg_option_str(cfg, 'pkg_name', + self.pkg_name) + self.conf_dir = util.get_cfg_option_str(cfg, 'config_dir', + self.conf_dir) + self.srv_name = util.get_cfg_option_str(cfg, 'service_name', + self.srv_name) def handle(name, cfg, cloud, log, _args): @@ -52,45 +84,49 @@ def handle(name, cfg, cloud, log, _args): " no 'salt_minion' key in configuration"), name) return - salt_cfg = cfg['salt_minion'] + s_cfg = cfg['salt_minion'] + const = SaltConstants(cfg=s_cfg) # Start by installing the salt package ... - cloud.distro.install_packages(('salt-minion',)) + cloud.distro.install_packages(const.pkg_name) # Ensure we can configure files at the right dir - config_dir = salt_cfg.get("config_dir", '/etc/salt') - util.ensure_dir(config_dir) + util.ensure_dir(const.conf_dir) # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - minion_data = util.yaml_dumps(salt_cfg.get('conf')) + if 'conf' in s_cfg: + # Add all sections from the conf object to minion config file + minion_config = os.path.join(const.conf_dir, 'minion') + minion_data = util.yaml_dumps(s_cfg.get('conf')) util.write_file(minion_config, minion_data) - if 'grains' in salt_cfg: + if 'grains' in s_cfg: # add grains to /etc/salt/grains - grains_config = os.path.join(config_dir, 'grains') - grains_data = util.yaml_dumps(salt_cfg.get('grains')) + grains_config = os.path.join(const.conf_dir, 'grains') + grains_data = util.yaml_dumps(s_cfg.get('grains')) util.write_file(grains_config, grains_data) # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - if os.path.isdir("/etc/salt/pki/minion"): - pki_dir_default = "/etc/salt/pki/minion" - else: - pki_dir_default = "/etc/salt/pki" + if 'public_key' in s_cfg and 'private_key' in s_cfg: + pki_dir_default = os.path.join(const.conf_dir, "pki/minion") + if not os.path.isdir(pki_dir_default): + pki_dir_default = os.path.join(const.conf_dir, "pki") - pki_dir = salt_cfg.get('pki_dir', pki_dir_default) + pki_dir = s_cfg.get('pki_dir', pki_dir_default) with util.umask(0o77): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') pem_name = os.path.join(pki_dir, 'minion.pem') - util.write_file(pub_name, salt_cfg['public_key']) - util.write_file(pem_name, salt_cfg['private_key']) + util.write_file(pub_name, s_cfg['public_key']) + util.write_file(pem_name, s_cfg['private_key']) + + # we need to have the salt minion service enabled in rc in order to be + # able to start the service. this does only apply on FreeBSD servers. + if cloud.distro.osfamily == 'freebsd': + cloud.distro.updatercconf('salt_minion_enable', 'YES') - # restart salt-minion. 'service' will start even if not started. if it + # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - util.subp(['service', 'salt-minion', 'restart'], capture=False) + util.subp(['service', const.srv_name, 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index fad11847..cf2e2409 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -113,9 +113,9 @@ cloud_final_modules: {% if variant not in ["freebsd"] %} - puppet - chef - - salt-minion - mcollective {% endif %} + - salt-minion - rightscale_userdata - scripts-vendor - scripts-per-once diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py index f13b48a0..70917a4c 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ b/tests/cloud_tests/testcases/modules/salt_minion.py @@ -31,4 +31,9 @@ class Test(base.CloudTestCase): out = self.get_data_file('grains') self.assertIn('role: web', out) + def test_minion_installed(self): + """Test if the salt-minion package is installed""" + out = self.get_data_file('minion_installed') + self.assertEqual(1, int(out)) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml index ab0e05bb..f20b9765 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -3,7 +3,7 @@ # # 2016-11-17: Currently takes >60 seconds results in test failure # -enabled: False +enabled: True cloud_config: | #cloud-config salt_minion: @@ -35,5 +35,8 @@ collect_scripts: grains: | #!/bin/bash cat /etc/salt/grains + minion_installed: | + #!/bin/bash + dpkg -l | grep salt-minion | grep ii | wc -l # vi: ts=4 expandtab -- cgit v1.2.3 From b7a790246e52520b47a467fe89b81199b9cf03f6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2018 04:22:24 +0100 Subject: shellify: raise TypeError on bad input. This makes 2 changes to shellify's behavior: a.) raise a TypeError rather than a RuntimeError. b.) raise a TypeError if input is not a list or tuple. --- cloudinit/tests/test_util.py | 23 ++++++++++++++++++++++ cloudinit/util.py | 14 +++++++++---- .../unittests/test_handler/test_handler_bootcmd.py | 7 ++++--- 3 files changed, 37 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index ba6bf699..c3e2e404 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -44,3 +44,26 @@ class TestUtil(CiTestCase): m_mount_info.return_value = ('/dev/sda1', 'btrfs', '/', 'ro,relatime') is_rw = util.mount_is_read_write('/') self.assertEqual(is_rw, False) + + +class TestShellify(CiTestCase): + + def test_input_dict_raises_type_error(self): + self.assertRaisesRegex( + TypeError, 'Input.*was.*dict.*xpected', + util.shellify, {'mykey': 'myval'}) + + def test_input_str_raises_type_error(self): + self.assertRaisesRegex( + TypeError, 'Input.*was.*str.*xpected', util.shellify, "foobar") + + def test_value_with_int_raises_type_error(self): + self.assertRaisesRegex( + TypeError, 'shellify.*int', util.shellify, ["foo", 1]) + + def test_supports_strings_and_lists(self): + self.assertEqual( + '\n'.join(["#!/bin/sh", "echo hi mom", "'echo' 'hi dad'", + "'echo' 'hi' 'sis'", ""]), + util.shellify(["echo hi mom", ["echo", "hi dad"], + ('echo', 'hi', 'sis')])) diff --git a/cloudinit/util.py b/cloudinit/util.py index 1d6cd1c5..083a8efe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1922,6 +1922,11 @@ def abs_join(*paths): # if it is an array, shell protect it (with single ticks) # if it is a string, do nothing def shellify(cmdlist, add_header=True): + if not isinstance(cmdlist, (tuple, list)): + raise TypeError( + "Input to shellify was type '%s'. Expected list or tuple." % + (type_utils.obj_name(cmdlist))) + content = '' if add_header: content += "#!/bin/sh\n" @@ -1930,7 +1935,7 @@ def shellify(cmdlist, add_header=True): for args in cmdlist: # If the item is a list, wrap all items in single tick. # If its not, then just write it directly. - if isinstance(args, list): + if isinstance(args, (list, tuple)): fixed = [] for f in args: fixed.append("'%s'" % (six.text_type(f).replace("'", escaped))) @@ -1940,9 +1945,10 @@ def shellify(cmdlist, add_header=True): content = "%s%s\n" % (content, args) cmds_made += 1 else: - raise RuntimeError(("Unable to shellify type %s" - " which is not a list or string") - % (type_utils.obj_name(args))) + raise TypeError( + "Unable to shellify type '%s'. Expected list, string, tuple. " + "Got: %s" % (type_utils.obj_name(args), args)) + LOG.debug("Shellified %s commands.", cmds_made) return content diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index dbf43e0d..09d4c681 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -69,7 +69,7 @@ class TestBootcmd(CiTestCase): cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) self.assertIn('Failed to shellify bootcmd', self.logs.getvalue()) self.assertEqual( - "'int' object is not iterable", + "Input to shellify was type 'int'. Expected list or tuple.", str(context_manager.exception)) @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") @@ -98,7 +98,7 @@ class TestBootcmd(CiTestCase): invalid_config = { 'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} cc = self._get_cloud('ubuntu') - with self.assertRaises(RuntimeError) as context_manager: + with self.assertRaises(TypeError) as context_manager: cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) expected_warnings = [ 'bootcmd.1: 20 is not valid under any of the given schemas', @@ -110,7 +110,8 @@ class TestBootcmd(CiTestCase): self.assertIn(warning, logs) self.assertIn('Failed to shellify', logs) self.assertEqual( - 'Unable to shellify type int which is not a list or string', + ("Unable to shellify type 'int'. Expected list, string, tuple. " + "Got: 20"), str(context_manager.exception)) def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self): -- cgit v1.2.3 From f891df345afa57c0c7734e8f04cca9a3d5881778 Mon Sep 17 00:00:00 2001 From: Douglas Jordan Date: Sat, 10 Mar 2018 07:20:08 +0100 Subject: This commit fixes get_hostname on the AzureDataSource. LP: #1754495 --- cloudinit/sources/DataSourceAzure.py | 2 ++ tests/unittests/test_datasource/test_azure.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 4bcbf3a4..0bb7fad9 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -223,6 +223,8 @@ DEF_PASSWD_REDACTION = 'REDACTED' def get_hostname(hostname_command='hostname'): + if not isinstance(hostname_command, (list, tuple)): + hostname_command = (hostname_command,) return util.subp(hostname_command, capture=True)[0].strip() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 254e9876..da7da0ca 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -643,6 +643,21 @@ fdescfs /dev/fd fdescfs rw 0 0 expected_config['config'].append(blacklist_config) self.assertEqual(netconfig, expected_config) + @mock.patch("cloudinit.sources.DataSourceAzure.util.subp") + def test_get_hostname_with_no_args(self, subp): + dsaz.get_hostname() + subp.assert_called_once_with(("hostname",), capture=True) + + @mock.patch("cloudinit.sources.DataSourceAzure.util.subp") + def test_get_hostname_with_string_arg(self, subp): + dsaz.get_hostname(hostname_command="hostname") + subp.assert_called_once_with(("hostname",), capture=True) + + @mock.patch("cloudinit.sources.DataSourceAzure.util.subp") + def test_get_hostname_with_iterable_arg(self, subp): + dsaz.get_hostname(hostname_command=("hostname",)) + subp.assert_called_once_with(("hostname",), capture=True) + class TestAzureBounce(CiTestCase): -- cgit v1.2.3 From 76460b63f9c310c7de4e5f0c11d1525bedd277e1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 14 Mar 2018 10:41:46 -0600 Subject: tests: Centralize and re-use skipTest based on json schema presense. This just centralizes a hunk of duplicated code and uses it from the new location. --- cloudinit/tests/helpers.py | 13 +++++++++++++ .../unittests/test_handler/test_handler_bootcmd.py | 12 +++--------- tests/unittests/test_handler/test_handler_ntp.py | 18 ++++++------------ .../unittests/test_handler/test_handler_resizefs.py | 14 +++----------- tests/unittests/test_handler/test_handler_runcmd.py | 14 ++++---------- tests/unittests/test_handler/test_schema.py | 21 +++++++-------------- 6 files changed, 36 insertions(+), 56 deletions(-) (limited to 'tests') diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 41d9a8ee..14c0b0bf 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -409,6 +409,19 @@ except AttributeError: return decorator +try: + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False +except ImportError: + _missing_jsonschema_dep = True + + +def skipUnlessJsonSchema(): + return skipIf( + _missing_jsonschema_dep, "No python-jsonschema dependency present.") + + # older versions of mock do not have the useful 'assert_not_called' if not hasattr(mock.Mock, 'assert_not_called'): def __mock_assert_not_called(mmock): diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index 09d4c681..29fc25e4 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -3,17 +3,11 @@ from cloudinit.config import cc_bootcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from cloudinit.tests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema import logging import tempfile -try: - import jsonschema - assert jsonschema # avoid pyflakes error F401: import unused - _missing_jsonschema_dep = False -except ImportError: - _missing_jsonschema_dep = True LOG = logging.getLogger(__name__) @@ -72,7 +66,7 @@ class TestBootcmd(CiTestCase): "Input to shellify was type 'int'. Expected list or tuple.", str(context_manager.exception)) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_type(self): """Schema validation warns of non-array type for bootcmd key. @@ -88,7 +82,7 @@ class TestBootcmd(CiTestCase): self.logs.getvalue()) self.assertIn('Failed to shellify', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency') + @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_item_type(self): """Schema validation warns of non-array or string bootcmd items. diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 28a8455d..695897c0 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,7 +3,8 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf +from cloudinit.tests.helpers import ( + FilesystemMockingTestCase, mock, skipUnlessJsonSchema) import os @@ -24,13 +25,6 @@ NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%} {% endif -%} """ -try: - import jsonschema - assert jsonschema # avoid pyflakes error F401: import unused - _missing_jsonschema_dep = False -except ImportError: - _missing_jsonschema_dep = True - class TestNtp(FilesystemMockingTestCase): @@ -312,7 +306,7 @@ class TestNtp(FilesystemMockingTestCase): content) self.assertNotIn('Invalid config:', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_ntp_handler_schema_validation_warns_non_string_item_type(self): """Ntp schema validation warns of non-strings in pools or servers. @@ -333,7 +327,7 @@ class TestNtp(FilesystemMockingTestCase): content = stream.read() self.assertEqual("servers ['valid', None]\npools [123]\n", content) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_ntp_handler_schema_validation_warns_of_non_array_type(self): """Ntp schema validation warns of non-array pools or servers types. @@ -354,7 +348,7 @@ class TestNtp(FilesystemMockingTestCase): content = stream.read() self.assertEqual("servers non-array\npools 123\n", content) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_ntp_handler_schema_validation_warns_invalid_key_present(self): """Ntp schema validation warns of invalid keys present in ntp config. @@ -378,7 +372,7 @@ class TestNtp(FilesystemMockingTestCase): "servers []\npools ['0.mycompany.pool.ntp.org']\n", content) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_ntp_handler_schema_validation_warns_of_duplicates(self): """Ntp schema validation warns of duplicates in servers or pools. diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 5aa3c498..c2a7f9fb 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -7,21 +7,13 @@ from collections import namedtuple import logging import textwrap -from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util, - wrap_and_call) +from cloudinit.tests.helpers import ( + CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) LOG = logging.getLogger(__name__) -try: - import jsonschema - assert jsonschema # avoid pyflakes error F401: import unused - _missing_jsonschema_dep = False -except ImportError: - _missing_jsonschema_dep = True - - class TestResizefs(CiTestCase): with_logs = True @@ -76,7 +68,7 @@ class TestResizefs(CiTestCase): 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self): """The handle reports json schema violations as a warning. diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 374c1d31..dbbb2717 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -3,19 +3,13 @@ from cloudinit.config import cc_runcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf +from cloudinit.tests.helpers import ( + FilesystemMockingTestCase, skipUnlessJsonSchema) import logging import os import stat -try: - import jsonschema - assert jsonschema # avoid pyflakes error F401: import unused - _missing_jsonschema_dep = False -except ImportError: - _missing_jsonschema_dep = True - LOG = logging.getLogger(__name__) @@ -56,7 +50,7 @@ class TestRuncmd(FilesystemMockingTestCase): ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_type(self): """Schema validation warns of non-array type for runcmd key. @@ -71,7 +65,7 @@ class TestRuncmd(FilesystemMockingTestCase): self.logs.getvalue()) self.assertIn('Failed to shellify', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency') + @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_item_type(self): """Schema validation warns of non-array or string runcmd items. diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index df67a0e0..1ecb6c68 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -6,7 +6,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, main) from cloudinit.util import subp, write_file -from cloudinit.tests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema from copy import copy import os @@ -14,13 +14,6 @@ from six import StringIO from textwrap import dedent from yaml import safe_load -try: - import jsonschema - assert jsonschema # avoid pyflakes error F401: import unused - _missing_jsonschema_dep = False -except ImportError: - _missing_jsonschema_dep = True - class GetSchemaTest(CiTestCase): @@ -73,7 +66,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase): with_logs = True - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_validateconfig_schema_non_strict_emits_warnings(self): """When strict is False validate_cloudconfig_schema emits warnings.""" schema = {'properties': {'p1': {'type': 'string'}}} @@ -82,7 +75,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase): "Invalid config:\np1: -1 is not of type 'string'\n", self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_validateconfig_schema_emits_warning_on_missing_jsonschema(self): """Warning from validate_cloudconfig_schema when missing jsonschema.""" schema = {'properties': {'p1': {'type': 'string'}}} @@ -92,7 +85,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase): 'Ignoring schema validation. python-jsonschema is not present', self.logs.getvalue()) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_validateconfig_schema_strict_raises_errors(self): """When strict is True validate_cloudconfig_schema raises errors.""" schema = {'properties': {'p1': {'type': 'string'}}} @@ -102,7 +95,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase): "Cloud config schema errors: p1: -1 is not of type 'string'", str(context_mgr.exception)) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_validateconfig_schema_honors_formats(self): """With strict True, validate_cloudconfig_schema errors on format.""" schema = { @@ -153,7 +146,7 @@ class ValidateCloudConfigFileTest(CiTestCase): self.config_file), str(context_mgr.exception)) - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_validateconfig_file_sctricty_validates_schema(self): """validate_cloudconfig_file raises errors on invalid schema.""" schema = { @@ -376,7 +369,7 @@ class CloudTestsIntegrationTest(CiTestCase): raises Warnings or errors on invalid cloud-config schema. """ - @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + @skipUnlessJsonSchema() def test_all_integration_test_cloud_config_schema(self): """Validate schema of cloud_tests yaml files looking for warnings.""" schema = get_schema() -- cgit v1.2.3 From 133ad2cb327ad17b7b81319fac8f9f14577c04df Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 14 Mar 2018 23:38:07 -0600 Subject: set_hostname: When present in metadata, set it before network bringup. When instance meta-data provides hostname information, run cc_set_hostname in the init-local or init-net stage before network comes up. Prevent an initial DHCP request which leaks the stock cloud-image default hostname before the meta-data provided hostname was processed. A leaked cloud-image hostname adversely affects Dynamic DNS which would reallocate 'ubuntu' hostname in DNS to every instance brought up by cloud-init. These instances would only update DNS to the cloud-init configured hostname upon DHCP lease renewal. This branch extends the get_hostname methods in datasource, cloud and util to limit results to metadata_only to avoid extra cost of querying the distro for hostname information if metadata does not provide that information. LP: #1746455 --- cloudinit/cloud.py | 5 +- cloudinit/cmd/main.py | 25 ++++ cloudinit/cmd/tests/test_main.py | 161 +++++++++++++++++++++ cloudinit/config/cc_set_hostname.py | 41 +++++- cloudinit/sources/__init__.py | 21 ++- cloudinit/sources/tests/test_init.py | 70 ++++++++- cloudinit/tests/test_util.py | 74 ++++++++++ cloudinit/util.py | 17 ++- .../test_handler/test_handler_set_hostname.py | 57 +++++++- 9 files changed, 449 insertions(+), 22 deletions(-) create mode 100644 cloudinit/cmd/tests/test_main.py (limited to 'tests') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index ba616781..6d12c437 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -78,8 +78,9 @@ class Cloud(object): def get_locale(self): return self.datasource.get_locale() - def get_hostname(self, fqdn=False): - return self.datasource.get_hostname(fqdn=fqdn) + def get_hostname(self, fqdn=False, metadata_only=False): + return self.datasource.get_hostname( + fqdn=fqdn, metadata_only=metadata_only) def device_name_to_device(self, name): return self.datasource.device_name_to_device(name) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index fcddd75c..3f2dbb93 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -40,6 +40,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, from cloudinit import atomic_helper +from cloudinit.config import cc_set_hostname from cloudinit.dhclient_hook import LogDhclient @@ -352,6 +353,11 @@ def main_init(name, args): LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) + if mode == sources.DSMODE_LOCAL: + # Before network comes up, set any configured hostname to allow + # dhcp clients to advertize this hostname to any DDNS services + # LP: #1746455. + _maybe_set_hostname(init, stage='local', retry_stage='network') init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: @@ -368,6 +374,7 @@ def main_init(name, args): init.setup_datasource() # update fully realizes user-data (pulling in #include if necessary) init.update() + _maybe_set_hostname(init, stage='init-net', retry_stage='modules:config') # Stage 7 try: # Attempt to consume the data per instance. @@ -681,6 +688,24 @@ def status_wrapper(name, args, data_d=None, link_d=None): return len(v1[mode]['errors']) +def _maybe_set_hostname(init, stage, retry_stage): + """Call set-hostname if metadata, vendordata or userdata provides it. + + @param stage: String representing current stage in which we are running. + @param retry_stage: String represented logs upon error setting hostname. + """ + cloud = init.cloudify() + (hostname, _fqdn) = util.get_hostname_fqdn( + init.cfg, cloud, metadata_only=True) + if hostname: # meta-data or user-data hostname content + try: + cc_set_hostname.handle('set-hostname', init.cfg, cloud, LOG, None) + except cc_set_hostname.SetHostnameError as e: + LOG.debug( + 'Failed setting hostname in %s stage. Will' + ' retry in %s stage. Error: %s.', stage, retry_stage, str(e)) + + def main_features(name, args): sys.stdout.write('\n'.join(sorted(version.FEATURES)) + '\n') diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py new file mode 100644 index 00000000..dbe421c0 --- /dev/null +++ b/cloudinit/cmd/tests/test_main.py @@ -0,0 +1,161 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from collections import namedtuple +import copy +import os +from six import StringIO + +from cloudinit.cmd import main +from cloudinit.util import ( + ensure_dir, load_file, write_file, yaml_dumps) +from cloudinit.tests.helpers import ( + FilesystemMockingTestCase, wrap_and_call) + +mypaths = namedtuple('MyPaths', 'run_dir') +myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand') + + +class TestMain(FilesystemMockingTestCase): + + with_logs = True + + def setUp(self): + super(TestMain, self).setUp() + self.new_root = self.tmp_dir() + self.cloud_dir = self.tmp_path('var/lib/cloud/', dir=self.new_root) + os.makedirs(self.cloud_dir) + self.replicateTestRoot('simple_ubuntu', self.new_root) + self.cfg = { + 'datasource_list': ['None'], + 'runcmd': ['ls /etc'], # test ALL_DISTROS + 'system_info': {'paths': {'cloud_dir': self.cloud_dir, + 'run_dir': self.new_root}}, + 'write_files': [ + { + 'path': '/etc/blah.ini', + 'content': 'blah', + 'permissions': 0o755, + }, + ], + 'cloud_init_modules': ['write-files', 'runcmd'], + } + cloud_cfg = yaml_dumps(self.cfg) + ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + self.cloud_cfg_file = os.path.join( + self.new_root, 'etc', 'cloud', 'cloud.cfg') + write_file(self.cloud_cfg_file, cloud_cfg) + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + self.stderr = StringIO() + self.patchStdoutAndStderr(stderr=self.stderr) + + def test_main_init_run_net_stops_on_file_no_net(self): + """When no-net file is present, main_init does not process modules.""" + stop_file = os.path.join(self.cloud_dir, 'data', 'no-net') # stop file + write_file(stop_file, '') + cmdargs = myargs( + debug=False, files=None, force=False, local=False, reporter=None, + subcommand='init') + (item1, item2) = wrap_and_call( + 'cloudinit.cmd.main', + {'util.close_stdin': True, + 'netinfo.debug_info': 'my net debug info', + 'util.fixup_output': ('outfmt', 'errfmt')}, + main.main_init, 'init', cmdargs) + # We should not run write_files module + self.assertFalse( + os.path.exists(os.path.join(self.new_root, 'etc/blah.ini')), + 'Unexpected run of write_files module produced blah.ini') + self.assertEqual([], item2) + # Instancify is called + instance_id_path = 'var/lib/cloud/data/instance-id' + self.assertFalse( + os.path.exists(os.path.join(self.new_root, instance_id_path)), + 'Unexpected call to datasource.instancify produced instance-id') + expected_logs = [ + "Exiting. stop file ['{stop_file}'] existed\n".format( + stop_file=stop_file), + 'my net debug info' # netinfo.debug_info + ] + for log in expected_logs: + self.assertIn(log, self.stderr.getvalue()) + + def test_main_init_run_net_runs_modules(self): + """Modules like write_files are run in 'net' mode.""" + cmdargs = myargs( + debug=False, files=None, force=False, local=False, reporter=None, + subcommand='init') + (item1, item2) = wrap_and_call( + 'cloudinit.cmd.main', + {'util.close_stdin': True, + 'netinfo.debug_info': 'my net debug info', + 'util.fixup_output': ('outfmt', 'errfmt')}, + main.main_init, 'init', cmdargs) + self.assertEqual([], item2) + # Instancify is called + instance_id_path = 'var/lib/cloud/data/instance-id' + self.assertEqual( + 'iid-datasource-none\n', + os.path.join(load_file( + os.path.join(self.new_root, instance_id_path)))) + # modules are run (including write_files) + self.assertEqual( + 'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini'))) + expected_logs = [ + 'network config is disabled by fallback', # apply_network_config + 'my net debug info', # netinfo.debug_info + 'no previous run detected' + ] + for log in expected_logs: + self.assertIn(log, self.stderr.getvalue()) + + def test_main_init_run_net_calls_set_hostname_when_metadata_present(self): + """When local-hostname metadata is present, call cc_set_hostname.""" + self.cfg['datasource'] = { + 'None': {'metadata': {'local-hostname': 'md-hostname'}}} + cloud_cfg = yaml_dumps(self.cfg) + write_file(self.cloud_cfg_file, cloud_cfg) + cmdargs = myargs( + debug=False, files=None, force=False, local=False, reporter=None, + subcommand='init') + + def set_hostname(name, cfg, cloud, log, args): + self.assertEqual('set-hostname', name) + updated_cfg = copy.deepcopy(self.cfg) + updated_cfg.update( + {'def_log_file': '/var/log/cloud-init.log', + 'log_cfgs': [], + 'syslog_fix_perms': ['syslog:adm', 'root:adm', 'root:wheel'], + 'vendor_data': {'enabled': True, 'prefix': []}}) + updated_cfg.pop('system_info') + + self.assertEqual(updated_cfg, cfg) + self.assertEqual(main.LOG, log) + self.assertIsNone(args) + + (item1, item2) = wrap_and_call( + 'cloudinit.cmd.main', + {'util.close_stdin': True, + 'netinfo.debug_info': 'my net debug info', + 'cc_set_hostname.handle': {'side_effect': set_hostname}, + 'util.fixup_output': ('outfmt', 'errfmt')}, + main.main_init, 'init', cmdargs) + self.assertEqual([], item2) + # Instancify is called + instance_id_path = 'var/lib/cloud/data/instance-id' + self.assertEqual( + 'iid-datasource-none\n', + os.path.join(load_file( + os.path.join(self.new_root, instance_id_path)))) + # modules are run (including write_files) + self.assertEqual( + 'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini'))) + expected_logs = [ + 'network config is disabled by fallback', # apply_network_config + 'my net debug info', # netinfo.debug_info + 'no previous run detected' + ] + for log in expected_logs: + self.assertIn(log, self.stderr.getvalue()) + +# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index aa3dfe5f..3d2b2da3 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -32,22 +32,51 @@ will be used. hostname: """ +import os + + +from cloudinit.atomic_helper import write_json from cloudinit import util +class SetHostnameError(Exception): + """Raised when the distro runs into an exception when setting hostname. + + This may happen if we attempt to set the hostname early in cloud-init's + init-local timeframe as certain services may not be running yet. + """ + pass + + def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) return - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + # Check for previous successful invocation of set-hostname + + # set-hostname artifact file accounts for both hostname and fqdn + # deltas. As such, it's format is different than cc_update_hostname's + # previous-hostname file which only contains the base hostname. + # TODO consolidate previous-hostname and set-hostname artifact files and + # distro._read_hostname implementation so we only validate one artifact. + prev_fn = os.path.join(cloud.get_cpath('data'), "set-hostname") + prev_hostname = {} + if os.path.exists(prev_fn): + prev_hostname = util.load_json(util.load_file(prev_fn)) + hostname_changed = (hostname != prev_hostname.get('hostname') or + fqdn != prev_hostname.get('fqdn')) + if not hostname_changed: + log.debug('No hostname changes. Skipping set-hostname') + return + log.debug("Setting the hostname to %s (%s)", fqdn, hostname) try: - log.debug("Setting the hostname to %s (%s)", fqdn, hostname) cloud.distro.set_hostname(hostname, fqdn) - except Exception: - util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn, - hostname) - raise + except Exception as e: + msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) + util.logexc(log, msg) + raise SetHostnameError("%s: %s" % (msg, e)) + write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index a05ca2f6..df0b374a 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -276,21 +276,34 @@ class DataSource(object): return "iid-datasource" return str(self.metadata['instance-id']) - def get_hostname(self, fqdn=False, resolve_ip=False): + def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): + """Get hostname or fqdn from the datasource. Look it up if desired. + + @param fqdn: Boolean, set True to return hostname with domain. + @param resolve_ip: Boolean, set True to attempt to resolve an ipv4 + address provided in local-hostname meta-data. + @param metadata_only: Boolean, set True to avoid looking up hostname + if meta-data doesn't have local-hostname present. + + @return: hostname or qualified hostname. Optionally return None when + metadata_only is True and local-hostname data is not available. + """ defdomain = "localdomain" defhost = "localhost" domain = defdomain if not self.metadata or 'local-hostname' not in self.metadata: + if metadata_only: + return None # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate # but instead, basically look up the existing hostname toks = [] hostname = util.get_hostname() - fqdn = util.get_fqdn_from_hosts(hostname) - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") + hosts_fqdn = util.get_fqdn_from_hosts(hostname) + if hosts_fqdn and hosts_fqdn.find(".") > 0: + toks = str(hosts_fqdn).split(".") elif hostname and hostname.find(".") > 0: toks = str(hostname).split(".") elif hostname: diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index af151154..5065083c 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -7,7 +7,7 @@ import stat from cloudinit.helpers import Paths from cloudinit.sources import ( INSTANCE_JSON_FILE, DataSource) -from cloudinit.tests.helpers import CiTestCase, skipIf +from cloudinit.tests.helpers import CiTestCase, skipIf, mock from cloudinit.user_data import UserDataProcessor from cloudinit import util @@ -108,6 +108,74 @@ class TestDataSource(CiTestCase): self.assertEqual('userdata_raw', datasource.userdata_raw) self.assertEqual('vendordata_raw', datasource.vendordata_raw) + def test_get_hostname_strips_local_hostname_without_domain(self): + """Datasource.get_hostname strips metadata local-hostname of domain.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp})) + self.assertTrue(datasource.get_data()) + self.assertEqual( + 'test-subclass-hostname', datasource.metadata['local-hostname']) + self.assertEqual('test-subclass-hostname', datasource.get_hostname()) + datasource.metadata['local-hostname'] = 'hostname.my.domain.com' + self.assertEqual('hostname', datasource.get_hostname()) + + def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self): + """Datasource.get_hostname with fqdn set gets qualified hostname.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp})) + self.assertTrue(datasource.get_data()) + datasource.metadata['local-hostname'] = 'hostname.my.domain.com' + self.assertEqual( + 'hostname.my.domain.com', datasource.get_hostname(fqdn=True)) + + def test_get_hostname_without_metadata_uses_system_hostname(self): + """Datasource.gethostname runs util.get_hostname when no metadata.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp})) + self.assertEqual({}, datasource.metadata) + mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts' + with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost: + with mock.patch(mock_fqdn) as m_fqdn: + m_gethost.return_value = 'systemhostname.domain.com' + m_fqdn.return_value = None # No maching fqdn in /etc/hosts + self.assertEqual('systemhostname', datasource.get_hostname()) + self.assertEqual( + 'systemhostname.domain.com', + datasource.get_hostname(fqdn=True)) + + def test_get_hostname_without_metadata_returns_none(self): + """Datasource.gethostname returns None when metadata_only and no MD.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp})) + self.assertEqual({}, datasource.metadata) + mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts' + with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost: + with mock.patch(mock_fqdn) as m_fqdn: + self.assertIsNone(datasource.get_hostname(metadata_only=True)) + self.assertIsNone( + datasource.get_hostname(fqdn=True, metadata_only=True)) + self.assertEqual([], m_gethost.call_args_list) + self.assertEqual([], m_fqdn.call_args_list) + + def test_get_hostname_without_metadata_prefers_etc_hosts(self): + """Datasource.gethostname prefers /etc/hosts to util.get_hostname.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp})) + self.assertEqual({}, datasource.metadata) + mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts' + with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost: + with mock.patch(mock_fqdn) as m_fqdn: + m_gethost.return_value = 'systemhostname.domain.com' + m_fqdn.return_value = 'fqdnhostname.domain.com' + self.assertEqual('fqdnhostname', datasource.get_hostname()) + self.assertEqual('fqdnhostname.domain.com', + datasource.get_hostname(fqdn=True)) + def test_get_data_write_json_instance_data(self): """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root.""" tmp = self.tmp_dir() diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index c3e2e404..d30643dc 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -16,6 +16,25 @@ MOUNT_INFO = [ ] +class FakeCloud(object): + + def __init__(self, hostname, fqdn): + self.hostname = hostname + self.fqdn = fqdn + self.calls = [] + + def get_hostname(self, fqdn=None, metadata_only=None): + myargs = {} + if fqdn is not None: + myargs['fqdn'] = fqdn + if metadata_only is not None: + myargs['metadata_only'] = metadata_only + self.calls.append(myargs) + if fqdn: + return self.fqdn + return self.hostname + + class TestUtil(CiTestCase): def test_parse_mount_info_no_opts_no_arg(self): @@ -67,3 +86,58 @@ class TestShellify(CiTestCase): "'echo' 'hi' 'sis'", ""]), util.shellify(["echo hi mom", ["echo", "hi dad"], ('echo', 'hi', 'sis')])) + + +class TestGetHostnameFqdn(CiTestCase): + + def test_get_hostname_fqdn_from_only_cfg_fqdn(self): + """When cfg only has the fqdn key, derive hostname and fqdn from it.""" + hostname, fqdn = util.get_hostname_fqdn( + cfg={'fqdn': 'myhost.domain.com'}, cloud=None) + self.assertEqual('myhost', hostname) + self.assertEqual('myhost.domain.com', fqdn) + + def test_get_hostname_fqdn_from_cfg_fqdn_and_hostname(self): + """When cfg has both fqdn and hostname keys, return them.""" + hostname, fqdn = util.get_hostname_fqdn( + cfg={'fqdn': 'myhost.domain.com', 'hostname': 'other'}, cloud=None) + self.assertEqual('other', hostname) + self.assertEqual('myhost.domain.com', fqdn) + + def test_get_hostname_fqdn_from_cfg_hostname_with_domain(self): + """When cfg has only hostname key which represents a fqdn, use that.""" + hostname, fqdn = util.get_hostname_fqdn( + cfg={'hostname': 'myhost.domain.com'}, cloud=None) + self.assertEqual('myhost', hostname) + self.assertEqual('myhost.domain.com', fqdn) + + def test_get_hostname_fqdn_from_cfg_hostname_without_domain(self): + """When cfg has a hostname without a '.' query cloud.get_hostname.""" + mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com') + hostname, fqdn = util.get_hostname_fqdn( + cfg={'hostname': 'myhost'}, cloud=mycloud) + self.assertEqual('myhost', hostname) + self.assertEqual('cloudhost.mycloud.com', fqdn) + self.assertEqual( + [{'fqdn': True, 'metadata_only': False}], mycloud.calls) + + def test_get_hostname_fqdn_from_without_fqdn_or_hostname(self): + """When cfg has neither hostname nor fqdn cloud.get_hostname.""" + mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com') + hostname, fqdn = util.get_hostname_fqdn(cfg={}, cloud=mycloud) + self.assertEqual('cloudhost', hostname) + self.assertEqual('cloudhost.mycloud.com', fqdn) + self.assertEqual( + [{'fqdn': True, 'metadata_only': False}, + {'metadata_only': False}], mycloud.calls) + + def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self): + """Calls to cloud.get_hostname pass the metadata_only parameter.""" + mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com') + hostname, fqdn = util.get_hostname_fqdn( + cfg={}, cloud=mycloud, metadata_only=True) + self.assertEqual( + [{'fqdn': True, 'metadata_only': True}, + {'metadata_only': True}], mycloud.calls) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 083a8efe..4504f053 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1025,9 +1025,16 @@ def dos2unix(contents): return contents.replace('\r\n', '\n') -def get_hostname_fqdn(cfg, cloud): - # return the hostname and fqdn from 'cfg'. If not found in cfg, - # then fall back to data from cloud +def get_hostname_fqdn(cfg, cloud, metadata_only=False): + """Get hostname and fqdn from config if present and fallback to cloud. + + @param cfg: Dictionary of merged user-data configuration (from init.cfg). + @param cloud: Cloud instance from init.cloudify(). + @param metadata_only: Boolean, set True to only query cloud meta-data, + returning None if not present in meta-data. + @return: a Tuple of strings , . Values can be none when + metadata_only is True and no cfg or metadata provides hostname info. + """ if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that fqdn = cfg['fqdn'] @@ -1041,11 +1048,11 @@ def get_hostname_fqdn(cfg, cloud): else: # no fqdn set, get fqdn from cloud. # get hostname from cfg if available otherwise cloud - fqdn = cloud.get_hostname(fqdn=True) + fqdn = cloud.get_hostname(fqdn=True, metadata_only=metadata_only) if "hostname" in cfg: hostname = cfg['hostname'] else: - hostname = cloud.get_hostname() + hostname = cloud.get_hostname(metadata_only=metadata_only) return (hostname, fqdn) diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index abdc17e7..d09ec23a 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -11,6 +11,7 @@ from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging +import os import shutil from six import BytesIO import tempfile @@ -19,14 +20,18 @@ LOG = logging.getLogger(__name__) class TestHostname(t_help.FilesystemMockingTestCase): + + with_logs = True + def setUp(self): super(TestHostname, self).setUp() self.tmp = tempfile.mkdtemp() + util.ensure_dir(os.path.join(self.tmp, 'data')) self.addCleanup(shutil.rmtree, self.tmp) def _fetch_distro(self, kind): cls = distros.fetch(kind) - paths = helpers.Paths({}) + paths = helpers.Paths({'cloud_dir': self.tmp}) return cls(kind, {}, paths) def test_write_hostname_rhel(self): @@ -34,7 +39,7 @@ class TestHostname(t_help.FilesystemMockingTestCase): 'hostname': 'blah.blah.blah.yahoo.com', } distro = self._fetch_distro('rhel') - paths = helpers.Paths({}) + paths = helpers.Paths({'cloud_dir': self.tmp}) ds = None cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) @@ -51,7 +56,7 @@ class TestHostname(t_help.FilesystemMockingTestCase): 'hostname': 'blah.blah.blah.yahoo.com', } distro = self._fetch_distro('debian') - paths = helpers.Paths({}) + paths = helpers.Paths({'cloud_dir': self.tmp}) ds = None cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) @@ -65,7 +70,7 @@ class TestHostname(t_help.FilesystemMockingTestCase): 'hostname': 'blah.blah.blah.suse.com', } distro = self._fetch_distro('sles') - paths = helpers.Paths({}) + paths = helpers.Paths({'cloud_dir': self.tmp}) ds = None cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) @@ -74,4 +79,48 @@ class TestHostname(t_help.FilesystemMockingTestCase): contents = util.load_file(distro.hostname_conf_fn) self.assertEqual('blah', contents.strip()) + def test_multiple_calls_skips_unchanged_hostname(self): + """Only new hostname or fqdn values will generate a hostname call.""" + distro = self._fetch_distro('debian') + paths = helpers.Paths({'cloud_dir': self.tmp}) + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + cc_set_hostname.handle( + 'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, []) + contents = util.load_file("/etc/hostname") + self.assertEqual('hostname1', contents.strip()) + cc_set_hostname.handle( + 'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, []) + self.assertIn( + 'DEBUG: No hostname changes. Skipping set-hostname\n', + self.logs.getvalue()) + cc_set_hostname.handle( + 'cc_set_hostname', {'hostname': 'hostname2.me.com'}, cc, LOG, []) + contents = util.load_file("/etc/hostname") + self.assertEqual('hostname2', contents.strip()) + self.assertIn( + 'Non-persistently setting the system hostname to hostname2', + self.logs.getvalue()) + + def test_error_on_distro_set_hostname_errors(self): + """Raise SetHostnameError on exceptions from distro.set_hostname.""" + distro = self._fetch_distro('debian') + + def set_hostname_error(hostname, fqdn): + raise Exception("OOPS on: %s" % fqdn) + + distro.set_hostname = set_hostname_error + paths = helpers.Paths({'cloud_dir': self.tmp}) + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + with self.assertRaises(cc_set_hostname.SetHostnameError) as ctx_mgr: + cc_set_hostname.handle( + 'somename', {'hostname': 'hostname1.me.com'}, cc, LOG, []) + self.assertEqual( + 'Failed to set the hostname to hostname1.me.com (hostname1):' + ' OOPS on: hostname1.me.com', + str(ctx_mgr.exception)) + # vi: ts=4 expandtab -- cgit v1.2.3 From bef2f2c945fdee4a1141c3177b3e48b1537027e4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Mar 2018 11:45:12 -0400 Subject: tests: Fix some warnings in tests that popped up with newer python. When running 'tox -e pylint' on a bionic system (python 3.6.4) I started seeing errors today like: tests/cloud_tests/platforms/__init__.py:5: [E0401(import-error), ] Unable to import 'tests.cloud_tests.platforms.ec2' The fix for those first errors was simply to create the __init__.py. The second set of changes fixes fallout found from actually now having pylint properly run on more of the cloud_tests. --- .pylintrc | 12 +++++++++++- tests/cloud_tests/platforms/ec2/__init__.py | 0 tests/cloud_tests/platforms/lxd/__init__.py | 0 tests/cloud_tests/platforms/lxd/platform.py | 4 ---- tests/cloud_tests/platforms/nocloudkvm/__init__.py | 0 tests/cloud_tests/platforms/nocloudkvm/instance.py | 2 +- tests/cloud_tests/platforms/nocloudkvm/platform.py | 4 ---- 7 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 tests/cloud_tests/platforms/ec2/__init__.py create mode 100644 tests/cloud_tests/platforms/lxd/__init__.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm/__init__.py (limited to 'tests') diff --git a/.pylintrc b/.pylintrc index 05a086d9..0bdfa59d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -46,7 +46,17 @@ reports=no # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=six.moves,pkg_resources,httplib,http.client,paramiko,simplestreams +ignored-modules= + http.client, + httplib, + pkg_resources, + six.moves, + # cloud_tests requirements. + boto3, + botocore, + paramiko, + pylxd, + simplestreams # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of diff --git a/tests/cloud_tests/platforms/ec2/__init__.py b/tests/cloud_tests/platforms/ec2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cloud_tests/platforms/lxd/__init__.py b/tests/cloud_tests/platforms/lxd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py index 6a016929..f7251a07 100644 --- a/tests/cloud_tests/platforms/lxd/platform.py +++ b/tests/cloud_tests/platforms/lxd/platform.py @@ -101,8 +101,4 @@ class LXDPlatform(Platform): """ return self.client.images.get_by_alias(alias) - def destroy(self): - """Clean up platform data.""" - super(LXDPlatform, self).destroy() - # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/__init__.py b/tests/cloud_tests/platforms/nocloudkvm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index 932dc0fa..33ff3f24 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -109,7 +109,7 @@ class NoCloudKVMInstance(Instance): if self.pid: try: c_util.subp(['kill', '-9', self.pid]) - except util.ProcessExectuionError: + except c_util.ProcessExecutionError: pass if self.pid_file: diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index a7e6f5de..85933463 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -21,10 +21,6 @@ class NoCloudKVMPlatform(Platform): platform_name = 'nocloud-kvm' - def __init__(self, config): - """Set up platform.""" - super(NoCloudKVMPlatform, self).__init__(config) - def get_image(self, img_conf): """Get image using specified image configuration. -- cgit v1.2.3 From 2f5d4cebb243cf2f30c665f034668ba4b14178f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Mar 2018 12:39:32 -0400 Subject: tests: fix run_tree and bddeb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was broken probably when we inserted the ssh keys into Platform.   tox -e citest tree_run and   tox -e citest bddeb would fail with KeyError in Platform.init due to lack of a data_dir. Also here are a few fixes found from attempting to make it work. --- tests/cloud_tests/bddeb.py | 2 +- tests/cloud_tests/platforms/platforms.py | 14 ++++++++++++-- tests/cloud_tests/util.py | 6 +++++- 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index a6d5069f..b9cfcfa6 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -16,7 +16,7 @@ pre_reqs = ['devscripts', 'equivs', 'git', 'tar'] def _out(cmd_res): """Get clean output from cmd result.""" - return cmd_res[0].strip() + return cmd_res[0].decode("utf-8").strip() def build_deb(args, instance): diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 1542b3be..abbfebba 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -2,12 +2,15 @@ """Base platform class.""" import os +import shutil from simplestreams import filters, mirrors from simplestreams import util as s_util from cloudinit import util as c_util +from tests.cloud_tests import util + class Platform(object): """Base class for platforms.""" @@ -17,7 +20,14 @@ class Platform(object): def __init__(self, config): """Set up platform.""" self.config = config - self._generate_ssh_keys(config['data_dir']) + self.tmpdir = util.mkdtemp() + if 'data_dir' in config: + self.data_dir = config['data_dir'] + else: + self.data_dir = os.path.join(self.tmpdir, "data_dir") + os.mkdir(self.data_dir) + + self._generate_ssh_keys(self.data_dir) def get_image(self, img_conf): """Get image using specified image configuration. @@ -29,7 +39,7 @@ class Platform(object): def destroy(self): """Clean up platform data.""" - pass + shutil.rmtree(self.tmpdir) def _generate_ssh_keys(self, data_dir): """Generate SSH keys to be used with image.""" diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 6ff285e7..3dd4996d 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -460,6 +460,10 @@ class PlatformError(IOError): IOError.__init__(self, message) +def mkdtemp(prefix='cloud_test_data'): + return tempfile.mkdtemp(prefix=prefix) + + class TempDir(object): """Configurable temporary directory like tempfile.TemporaryDirectory.""" @@ -480,7 +484,7 @@ class TempDir(object): @return_value: tempdir path """ if not self.tmpdir: - self.tmpdir = tempfile.mkdtemp(prefix=self.prefix) + self.tmpdir = mkdtemp(prefix=self.prefix) LOG.debug('using tmpdir: %s', self.tmpdir) return self.tmpdir -- cgit v1.2.3 From 97012fbb5207ddf1d2dcbb1c5eae710c47ab8ec0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 15 Mar 2018 11:40:13 -0600 Subject: util: Fix subp regression. Allow specifying subp command as a string. The command provided to subp can either be a string or a list. This patch fixes a regression which raised CalledProcessError whenever providing a string to subp. LP: #1755965 --- cloudinit/util.py | 10 ++++++++-- tests/unittests/test_util.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/cloudinit/util.py b/cloudinit/util.py index 4504f053..823d80bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1874,8 +1874,14 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, # Popen converts entries in the arguments array from non-bytes to bytes. # When locale is unset it may use ascii for that encoding which can # cause UnicodeDecodeErrors. (LP: #1751051) - bytes_args = [x if isinstance(x, six.binary_type) else x.encode("utf-8") - for x in args] + if isinstance(args, six.binary_type): + bytes_args = args + elif isinstance(args, six.string_types): + bytes_args = args.encode("utf-8") + else: + bytes_args = [ + x if isinstance(x, six.binary_type) else x.encode("utf-8") + for x in args] try: sp = subprocess.Popen(bytes_args, stdout=stdout, stderr=stderr, stdin=stdin, diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 89ae40f5..499e7c9f 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -632,6 +632,24 @@ class TestSubp(helpers.CiTestCase): # but by using bash, we remove dependency on another program. return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) + def test_subp_handles_bytestrings(self): + """subp can run a bytestring command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = util.subp(cmd.encode('utf-8'), shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_strings(self): + """subp can run a string command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = util.subp(cmd, shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + def test_subp_handles_utf8(self): # The given bytes contain utf-8 accented characters as seen in e.g. # the "deja dup" package in Ubuntu. -- cgit v1.2.3 From b7b7331b9c308d8e1cb0b3dfd4398e6e7cb1b60f Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 16 Mar 2018 14:00:15 -0400 Subject: netplan: render bridge port-priority values Update netplan renderer to write out bridge port-priority values now that netplan supports the feature. LP: #1735821 --- cloudinit/net/netplan.py | 10 +++++----- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index d3788af8..6bee3d37 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -311,12 +311,12 @@ class Renderer(renderer.Renderer): if newname is None: continue br_config.update({newname: value}) - if newname == 'path-cost': - # -> : int() + if newname in ['path-cost', 'port-priority']: + # -> : int() newvalue = {} - for costval in value: - (port, cost) = costval.split() - newvalue[port] = int(cost) + for val in value: + (port, portval) = val.split() + newvalue[port] = int(portval) br_config.update({newname: newvalue}) if len(br_config) > 0: diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index fe667d88..1dd7ded7 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -47,7 +47,7 @@ NET_CONFIG_TO_V2 = { 'bridge_maxage': 'max-age', 'bridge_maxwait': None, 'bridge_pathcost': 'path-cost', - 'bridge_portprio': None, + 'bridge_portprio': 'port-priority', 'bridge_stp': 'stp', 'bridge_waitport': None}} diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ac33e8ef..9cf11f2e 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -758,6 +758,9 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true path-cost: eth3: 50 eth4: 75 + port-priority: + eth3: 28 + eth4: 14 priority: 22 stp: false routes: -- cgit v1.2.3 From 95bb226921b8075ca9f65a9d2b672a3e342498b7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Mar 2018 13:41:40 -0600 Subject: tests: Make pylint happy and fix python2.6 uses of assertRaisesRegex. Older unittest2.TestCase (as seen in CentOS 6) do not have an assertRaisesRegex method. They only have the now-deprecated assertRaisesRegexp. We need our unit tests to work there and on newer python (3.6). Simply making assertRaisesRegex = assertRaisesRegexp makes pylint complain as described in https://github.com/PyCQA/pylint/issues/1946 . What was here before this commit was actually broken. This commit makes assertRaisesRegex functional in CentOS 6 and works around the invalid Deprecated warning from pylint. To prove this, we use assertRaisesRegex in a unit test which will be exectued in py27, py3 and py26. --- cloudinit/tests/helpers.py | 12 ++++++------ tests/unittests/test_handler/test_handler_apt_source_v1.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 14c0b0bf..a2e10536 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -433,12 +433,12 @@ if not hasattr(mock.Mock, 'assert_not_called'): mock.Mock.assert_not_called = __mock_assert_not_called -# older unittest2.TestCase (centos6) do not have assertRaisesRegex -# And setting assertRaisesRegex to assertRaisesRegexp causes -# https://github.com/PyCQA/pylint/issues/1653 . So the workaround. +# older unittest2.TestCase (centos6) have only the now-deprecated +# assertRaisesRegexp. Simple assignment makes pylint complain, about +# users of assertRaisesRegex so we use getattr to trick it. +# https://github.com/PyCQA/pylint/issues/1946 if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): - def _tricky(*args, **kwargs): - return unittest2.TestCase.assertRaisesRegexp - unittest2.TestCase.assertRaisesRegex = _tricky + unittest2.TestCase.assertRaisesRegex = ( + getattr(unittest2.TestCase, 'assertRaisesRegexp')) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 3a3f95ca..46ca4ce4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -569,7 +569,8 @@ class TestAptSourceConfig(TestCase): newcfg = cc_apt_configure.convert_to_v3_apt_format(cfg_3_only) self.assertEqual(newcfg, cfg_3_only) # collision (unequal) - with self.assertRaises(ValueError): + match = "Old and New.*unequal.*apt_proxy" + with self.assertRaisesRegex(ValueError, match): cc_apt_configure.convert_to_v3_apt_format(cfgconflict) def test_convert_to_new_format_dict_collision(self): -- cgit v1.2.3 From a1f678f8ebc080d4737f32275f42947b84ae025a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 16 Mar 2018 13:43:31 -0600 Subject: cc_snap: Add new module to install and configure snapd and snap packages. Support installing and configuring snaps on ubuntu systems. Now, cloud-config files can provide a list or dictionary of snap:assertions which will be allow configuration of snapd on a system via 'snap ack' calls. The snap:commands configuration option supports arbitrary system commands intended to interact with snappy's cli. This allows users to run arbitrary snappy commands to create users, download, install and configure snap packages and snapd. This branch also deprecates old snappy and snap_config modules leaving warnings in documentation and runtime for consumers of these modules. Deprecated snap* modules will be dropped in cloud-init v.18.2 release. --- cloudinit/config/cc_puppet.py | 8 +- cloudinit/config/cc_snap.py | 273 +++++++++++ cloudinit/config/cc_snap_config.py | 7 + cloudinit/config/cc_snappy.py | 8 + cloudinit/config/tests/test_snap.py | 533 +++++++++++++++++++++ cloudinit/util.py | 13 +- config/cloud.cfg.tmpl | 5 +- doc/rtd/conf.py | 1 + doc/rtd/topics/modules.rst | 1 + tests/cloud_tests/releases.yaml | 3 + tests/cloud_tests/testcases.yaml | 3 + tests/cloud_tests/testcases/__init__.py | 3 + tests/cloud_tests/testcases/base.py | 173 ++++++- .../testcases/main/command_output_simple.py | 17 +- tests/cloud_tests/testcases/modules/snap.py | 16 + tests/cloud_tests/testcases/modules/snap.yaml | 18 + tests/cloud_tests/testcases/modules/snappy.py | 2 + tests/cloud_tests/verify.py | 11 +- tests/unittests/test_handler/test_schema.py | 1 + tests/unittests/test_util.py | 33 ++ 20 files changed, 1098 insertions(+), 31 deletions(-) create mode 100644 cloudinit/config/cc_snap.py create mode 100644 cloudinit/config/tests/test_snap.py create mode 100644 tests/cloud_tests/testcases/modules/snap.py create mode 100644 tests/cloud_tests/testcases/modules/snap.yaml (limited to 'tests') diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 57a170fb..297e0721 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -140,8 +140,9 @@ def handle(name, cfg, cloud, log, _args): # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) - puppet_config.readfp(StringIO(cleaned_contents), - filename=p_constants.conf_path) + puppet_config.readfp( # pylint: disable=W1505 + StringIO(cleaned_contents), + filename=p_constants.conf_path) for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place @@ -149,8 +150,7 @@ def handle(name, cfg, cloud, log, _args): # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership util.ensure_dir(p_constants.ssl_dir, 0o771) - util.chownbyname(p_constants.ssl_dir, 'puppet', 'root') - util.ensure_dir(p_constants.ssl_cert_dir) + util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root') util.write_file(p_constants.ssl_cert_path, cfg) util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py new file mode 100644 index 00000000..db965291 --- /dev/null +++ b/cloudinit/config/cc_snap.py @@ -0,0 +1,273 @@ +# Copyright (C) 2018 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Snap: Install, configure and manage snapd and snap packages.""" + +import sys +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE +from cloudinit import util + + +distros = ['ubuntu'] +frequency = PER_INSTANCE + +LOG = logging.getLogger(__name__) + +schema = { + 'id': 'cc_snap', + 'name': 'Snap', + 'title': 'Install, configure and manage snapd and snap packages', + 'description': dedent("""\ + This module provides a simple configuration namespace in cloud-init to + both setup snapd and install snaps. + + .. note:: + Both ``assertions`` and ``commands`` values can be either a + dictionary or a list. If these configs are provided as a + dictionary, the keys are only used to order the execution of the + assertions or commands and the dictionary is merged with any + vendor-data snap configuration provided. If a list is provided by + the user instead of a dict, any vendor-data snap configuration is + ignored. + + The ``assertions`` configuration option is a dictionary or list of + properly-signed snap assertions which will run before any snap + ``commands``. They will be added to snapd's assertion database by + invoking ``snap ack ``. + + Snap ``commands`` is a dictionary or list of individual snap + commands to run on the target system. These commands can be used to + create snap users, install snaps and provide snap configuration. + + .. note:: + If 'side-loading' private/unpublished snaps on an instance, it is + best to create a snap seed directory and seed.yaml manifest in + **/var/lib/snapd/seed/** which snapd automatically installs on + startup. + + **Development only**: The ``squashfuse_in_container`` boolean can be + set true to install squashfuse package when in a container to enable + snap installs. Default is false. + """), + 'distros': distros, + 'examples': [dedent("""\ + snap: + assertions: + 00: | + signed_assertion_blob_here + 02: | + signed_assertion_blob_here + commands: + 00: snap create-user --sudoer --known @mydomain.com + 01: snap install canonical-livepatch + 02: canonical-livepatch enable + """), dedent("""\ + # LXC-based containers require squashfuse before snaps can be installed + snap: + commands: + 00: apt-get install squashfuse -y + 11: snap install emoj + + """), dedent("""\ + # Convenience: the snap command can be omitted when specifying commands + # as a list and 'snap' will automatically be prepended. + # The following commands are equivalent: + snap: + commands: + 00: ['install', 'vlc'] + 01: ['snap', 'install', 'vlc'] + 02: snap install vlc + 03: 'snap install vlc' + """)], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'snap': { + 'type': 'object', + 'properties': { + 'assertions': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': {'type': 'string'}, + 'additionalItems': False, # Reject items non-string + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + }, + 'commands': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject non-string & non-list + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + }, + 'squashfuse_in_container': { + 'type': 'boolean' + } + }, + 'additionalProperties': False, # Reject keys not in schema + 'required': [], + 'minProperties': 1 + } + } +} + +# TODO schema for 'assertions' and 'commands' are too permissive at the moment. +# Once python-jsonschema supports schema draft 6 add support for arbitrary +# object keys with 'patternProperties' constraint to validate string values. + +__doc__ = get_schema_doc(schema) # Supplement python help() + +SNAP_CMD = "snap" +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" + + +def add_assertions(assertions): + """Import list of assertions. + + Import assertions by concatenating each assertion into a + string separated by a '\n'. Write this string to a instance file and + then invoke `snap ack /path/to/file` and check for errors. + If snap exits 0, then all assertions are imported. + """ + if not assertions: + return + LOG.debug('Importing user-provided snap assertions') + if isinstance(assertions, dict): + assertions = assertions.values() + elif not isinstance(assertions, list): + raise TypeError( + 'assertion parameter was not a list or dict: {assertions}'.format( + assertions=assertions)) + + snap_cmd = [SNAP_CMD, 'ack'] + combined = "\n".join(assertions) + + for asrt in assertions: + LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) + + util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) + util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + + +def prepend_snap_commands(commands): + """Ensure user-provided commands start with SNAP_CMD, warn otherwise. + + Each command is either a list or string. Perform the following: + - When the command is a list, pop the first element if it is None + - When the command is a list, insert SNAP_CMD as the first element if + not present. + - When the command is a string containing a non-snap command, warn. + + Support cut-n-paste snap command sets from public snappy documentation. + Allow flexibility to provide non-snap environment/config setup if needed. + + @commands: List of commands. Each command element is a list or string. + + @return: List of 'fixed up' snap commands. + @raise: TypeError on invalid config item type. + """ + warnings = [] + errors = [] + fixed_commands = [] + for command in commands: + if isinstance(command, list): + if command[0] is None: # Avoid warnings by specifying None + command = command[1:] + elif command[0] != SNAP_CMD: # Automatically prepend SNAP_CMD + command.insert(0, SNAP_CMD) + elif isinstance(command, str): + if not command.startswith('%s ' % SNAP_CMD): + warnings.append(command) + else: + errors.append(str(command)) + continue + fixed_commands.append(command) + + if warnings: + LOG.warning( + 'Non-snap commands in snap config:\n%s', '\n'.join(warnings)) + if errors: + raise TypeError( + 'Invalid snap config.' + ' These commands are not a string or list:\n' + '\n'.join(errors)) + return fixed_commands + + +def run_commands(commands): + """Run the provided commands provided in snap:commands configuration. + + Commands are run individually. Any errors are collected and reported + after attempting all commands. + + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ + if not commands: + return + LOG.debug('Running user-provided snap commands') + if isinstance(commands, dict): + # Sort commands based on dictionary key + commands = [v for _, v in sorted(commands.items())] + elif not isinstance(commands, list): + raise TypeError( + 'commands parameter was not a list or dict: {commands}'.format( + commands=commands)) + + fixed_snap_commands = prepend_snap_commands(commands) + + cmd_failures = [] + for command in fixed_snap_commands: + shell = isinstance(command, str) + try: + util.subp(command, shell=shell, status_cb=sys.stderr.write) + except util.ProcessExecutionError as e: + cmd_failures.append(str(e)) + if cmd_failures: + msg = 'Failures running snap commands:\n{cmd_failures}'.format( + cmd_failures=cmd_failures) + util.logexc(LOG, msg) + raise RuntimeError(msg) + + +# RELEASE_BLOCKER: Once LP: #1628289 is released on xenial, drop this function. +def maybe_install_squashfuse(cloud): + """Install squashfuse if we are in a container.""" + if not util.is_container(): + return + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(LOG, "Package update failed") + raise + try: + cloud.distro.install_packages(['squashfuse']) + except Exception as e: + util.logexc(LOG, "Failed to install squashfuse") + raise + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snap', {}) + if not cfgin: + LOG.debug(("Skipping module named %s," + " no 'snap' key in configuration"), name) + return + + validate_cloudconfig_schema(cfg, schema) + if util.is_true(cfgin.get('squashfuse_in_container', False)): + maybe_install_squashfuse(cloud) + add_assertions(cfgin.get('assertions', [])) + run_commands(cfgin.get('commands', [])) + +# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py index e82c0811..afe297ee 100644 --- a/cloudinit/config/cc_snap_config.py +++ b/cloudinit/config/cc_snap_config.py @@ -4,11 +4,15 @@ # # This file is part of cloud-init. See LICENSE file for license information. +# RELEASE_BLOCKER: Remove this deprecated module in 18.3 """ Snap Config ----------- **Summary:** snap_config modules allows configuration of snapd. +**Deprecated**: Use :ref:`snap` module instead. This module will not exist +in cloud-init 18.3. + This module uses the same ``snappy`` namespace for configuration but acts only only a subset of the configuration. @@ -154,6 +158,9 @@ def handle(name, cfg, cloud, log, args): LOG.debug('No snappy config provided, skipping') return + log.warning( + 'DEPRECATION: snap_config module will be dropped in 18.3 release.' + ' Use snap module instead') if not(util.system_is_snappy()): LOG.debug("%s: system not snappy", name) return diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index eecb8178..bab80bbe 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,10 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. +# RELEASE_BLOCKER: Remove this deprecated module in 18.3 """ Snappy ------ **Summary:** snappy modules allows configuration of snappy. +**Deprecated**: Use :ref:`snap` module instead. This module will not exist +in cloud-init 18.3. + The below example config config would install ``etcd``, and then install ``pkg2.smoser`` with a ```` argument where ``config-file`` has ``config-blob`` inside it. If ``pkgname`` is installed already, then @@ -271,6 +275,10 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return + log.warning( + 'DEPRECATION: snappy module will be dropped in 18.3 release.' + ' Use snap module instead') + set_snappy_command() pkg_ops = get_package_ops(packages=mycfg['packages'], diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py new file mode 100644 index 00000000..c2dd6afe --- /dev/null +++ b/cloudinit/config/tests/test_snap.py @@ -0,0 +1,533 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re +from six import StringIO + +from cloudinit.config.cc_snap import ( + ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands, + maybe_install_squashfuse, run_commands, schema) +from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call + + +SYSTEM_USER_ASSERTION = """\ +type: system-user +authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +email: foo@bar.com +password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt +series: +- 16 +since: 2016-09-10T16:34:00+03:00 +until: 2017-11-10T16:34:00+03:00 +username: baz +sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj + +AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP +Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI +zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF +s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj ++to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP +Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS +d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q +BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H +f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V +v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""" + +ACCOUNT_ASSERTION = """\ +type: account-key +authority-id: canonical +revision: 2 +public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 +account-id: canonical +name: store +since: 2016-04-01T00:00:00.0Z +body-length: 717 +sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH + +AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j +qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 +vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ +UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK +Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG +o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl +VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 +2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an +Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc +vUvV7RjVzv17ut0AEQEAAQ== + +AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM +WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b +nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL +3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL +eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY +inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 +rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ +rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE +aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ +6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO +haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF +yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 +HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi +skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK +CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde +ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF +qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR +IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t +oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""" + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestAddAssertions(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestAddAssertions, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_on_empty_list(self, m_subp): + """When provided with an empty list, add_assertions does nothing.""" + add_assertions([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_add_assertions_on_non_list_or_dict(self): + """When provided an invalid type, add_assertions raises an error.""" + with self.assertRaises(TypeError) as context_manager: + add_assertions(assertions="I'm Not Valid") + self.assertEqual( + "assertion parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_list(self, m_subp): + """When provided with a list, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION] + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + 'sertions', self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + util.write_file(compare_file, '\n'.join(assertions).encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_dict(self, m_subp): + """When provided with a dict, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = {'00': SYSTEM_USER_ASSERTION, '01': ACCOUNT_ASSERTION} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: system-user', 'authority-id: Lqv", + self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: account-key', 'authority-id: canonic", + self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + combined = '\n'.join(assertions.values()) + util.write_file(compare_file, combined.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + +class TestPrepentSnapCommands(CiTestCase): + + with_logs = True + + def test_prepend_snap_commands_errors_on_neither_string_nor_list(self): + """Raise an error for each command which is not a string or list.""" + orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']] + with self.assertRaises(TypeError) as context_manager: + prepend_snap_commands(orig_commands) + self.assertEqual( + "Invalid snap config. These commands are not a string or list:\n" + "1\n{'not': 'gonna work'}", + str(context_manager.exception)) + + def test_prepend_snap_commands_warns_on_non_snap_string_commands(self): + """Warn on each non-snap for commands of type string.""" + orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x'] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual( + 'WARNING: Non-snap commands in snap config:\n' + 'ls\ntouch /blah\n', + self.logs.getvalue()) + self.assertEqual(orig_commands, fixed_commands) + + def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self): + """Prepend 'snap' for each non-snap command of type list.""" + orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'], + ['snap', 'install', 'x']] + expected = [['snap', 'ls'], ['snap', 'list'], + ['snap', 'snapa', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + def test_prepend_snap_commands_removes_first_item_when_none(self): + """Remove the first element of a non-snap command when it is None.""" + orig_commands = [[None, 'ls'], ['snap', 'list'], + [None, 'touch', '/blah'], + ['snap', 'install', 'x']] + expected = [['ls'], ['snap', 'list'], + ['touch', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + +class TestRunCommands(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestRunCommands, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_run_commands_on_empty_list(self, m_subp): + """When provided with an empty list, run_commands does nothing.""" + run_commands([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_run_commands_on_non_list_or_dict(self): + """When provided an invalid type, run_commands raises an error.""" + with self.assertRaises(TypeError) as context_manager: + run_commands(commands="I'm Not Valid") + self.assertEqual( + "commands parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + def test_run_command_logs_commands_and_exit_codes_to_stderr(self): + """All exit codes are logged to stderr.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'bogus command' + cmd3 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2, cmd3] + + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with self.assertRaises(RuntimeError) as context_manager: + run_commands(commands=commands) + + self.assertIsNotNone( + re.search(r'bogus: (command )?not found', + str(context_manager.exception)), + msg='Expected bogus command not found') + expected_stderr_log = '\n'.join([ + 'Begin run command: {cmd}'.format(cmd=cmd1), + 'End run command: exit(0)', + 'Begin run command: {cmd}'.format(cmd=cmd2), + 'ERROR: End run command: exit(127)', + 'Begin run command: {cmd}'.format(cmd=cmd3), + 'End run command: exit(0)\n']) + self.assertEqual(expected_stderr_log, m_stderr.getvalue()) + + def test_run_command_as_lists(self): + """When commands are specified as a list, run them in order.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2] + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + self.assertIn( + 'DEBUG: Running user-provided snap commands', + self.logs.getvalue()) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + self.assertIn( + 'WARNING: Non-snap commands in snap config:', self.logs.getvalue()) + + def test_run_command_dict_sorted_as_command_script(self): + """When commands are a dict, sort them and run.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = {'02': cmd1, '01': cmd2} + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + expected_messages = [ + 'DEBUG: Running user-provided snap commands'] + for message in expected_messages: + self.assertIn(message, self.logs.getvalue()) + self.assertEqual('MOM\nHI\n', util.load_file(outfile)) + + +class TestSchema(CiTestCase): + + with_logs = True + + def test_schema_warns_on_snap_not_as_dict(self): + """If the snap configuration is not a dict, emit a warning.""" + validate_cloudconfig_schema({'snap': 'wrong type'}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap: 'wrong type' is not of type" + " 'object'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_disallows_unknown_keys(self, _): + """Unknown keys in the snap configuration emit warnings.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['ls'], 'invalid-key': ''}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: Additional properties are not' + " allowed ('invalid-key' was unexpected)", + self.logs.getvalue()) + + def test_warn_schema_requires_either_commands_or_assertions(self): + """Warn when snap configuration lacks both commands and assertions.""" + validate_cloudconfig_schema( + {'snap': {}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: {} does not have enough' + ' properties', + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_commands_is_not_list_or_dict(self, _): + """Warn when snap:commands config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: 'broken' is not of type" + " 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_when_commands_is_empty(self, _): + """Emit warnings when snap:commands is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: [] is too short\n" + "WARNING: Invalid config:\nsnap.commands: {} does not have enough" + " properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_are_list_or_dict(self, _): + """No warnings when snap:commands are either a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_assertions_is_not_list_or_dict(self, _): + """Warn when snap:assertions config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: 'broken' is not of" + " type 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_when_assertions_is_empty(self, _): + """Emit warnings when snap:assertions is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: [] is too short\n" + "WARNING: Invalid config:\nsnap.assertions: {} does not have" + " enough properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_schema_when_assertions_are_list_or_dict(self, _): + """No warnings when snap:assertions are a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + +class TestHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestHandle, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.validate_cloudconfig_schema') + def test_handle_no_config(self, m_schema, m_add, m_run): + """When no snap-related configuration is provided, nothing happens.""" + cfg = {} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertIn( + "DEBUG: Skipping module named snap, no 'snap' key in config", + self.logs.getvalue()) + m_schema.assert_not_called() + m_add.assert_not_called() + m_run.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_skips_squashfuse_when_unconfigured(self, m_squash, m_add, + m_run): + """When squashfuse_in_container is unset, don't attempt to install.""" + handle( + 'snap', cfg={'snap': {}}, cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': None}}, + cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': False}}, + cloud=None, log=self.logger, args=None) + self.assertEqual([], m_squash.call_args_list) # No calls + # snap configuration missing assertions and commands will default to [] + self.assertIn(mock.call([]), m_add.call_args_list) + self.assertIn(mock.call([]), m_run.call_args_list) + + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_tries_to_install_squashfuse(self, m_squash): + """If squashfuse_in_container is True, try installing squashfuse.""" + cfg = {'snap': {'squashfuse_in_container': True}} + mycloud = FakeCloud(None) + handle('snap', cfg=cfg, cloud=mycloud, log=self.logger, args=None) + self.assertEqual( + [mock.call(mycloud)], m_squash.call_args_list) + + def test_handle_runs_commands_provided(self): + """If commands are specified as a list, run them.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cfg = { + 'snap': {'commands': ['echo "HI" >> %s' % outfile, + 'echo "MOM" >> %s' % outfile]}} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_adds_assertions(self, m_subp): + """Any configured snap assertions are provided to add_assertions.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + compare_file = self.tmp_path('comparison', dir=self.tmp) + cfg = { + 'snap': {'assertions': [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]}} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + content = '\n'.join(cfg['snap']['assertions']) + util.write_file(compare_file, content.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_validates_schema(self, m_subp): + """Any provided configuration is runs validate_cloudconfig_schema.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + cfg = {'snap': {'invalid': ''}} # Generates schema warning + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual( + "WARNING: Invalid config:\nsnap: Additional properties are not" + " allowed ('invalid' was unexpected)\n", + self.logs.getvalue()) + + +class TestMaybeInstallSquashFuse(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestMaybeInstallSquashFuse, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_skips_non_containers(self, m_container): + """maybe_install_squashfuse does nothing when not on a container.""" + m_container.return_value = False + maybe_install_squashfuse(cloud=FakeCloud(None)) + self.assertEqual([mock.call()], m_container.call_args_list) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_install_errors(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_update_errors(self, m_container): + """maybe_install_squashfuse logs and raises package update errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_happy_path(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() # No errors raised + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual( + [mock.call()], distro.update_package_sources.call_args_list) + self.assertEqual( + [mock.call(['squashfuse'])], + distro.install_packages.call_args_list) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 823d80bf..cae8b196 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1827,7 +1827,8 @@ def subp_blob_in_tempfile(blob, *args, **kwargs): def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, - logstring=False, decode="replace", target=None, update_env=None): + logstring=False, decode="replace", target=None, update_env=None, + status_cb=None): # not supported in cloud-init (yet), for now kept in the call signature # to ease maintaining code shared between cloud-init and curtin @@ -1848,6 +1849,9 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, if target_path(target) != "/": args = ['chroot', target] + list(args) + if status_cb: + command = ' '.join(args) if isinstance(args, list) else args + status_cb('Begin run command: {command}\n'.format(command=command)) if not logstring: LOG.debug(("Running command %s with allowed return codes %s" " (shell=%s, capture=%s)"), args, rcs, shell, capture) @@ -1888,6 +1892,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, env=env, shell=shell) (out, err) = sp.communicate(data) except OSError as e: + if status_cb: + status_cb('ERROR: End run command: invalid command provided\n') raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", @@ -1912,9 +1918,14 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, rc = sp.returncode if rc not in rcs: + if status_cb: + status_cb( + 'ERROR: End run command: exit({code})\n'.format(code=rc)) raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, cmd=args) + if status_cb: + status_cb('End run command: exit({code})\n'.format(code=rc)) return (out, err) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index cf2e2409..56a34fab 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -72,7 +72,8 @@ cloud_config_modules: # Emit the cloud config ready event # this can be used by upstart jobs for 'start on cloud-config'. - emit_upstart - - snap_config + - snap + - snap_config # DEPRECATED- Drop in version 18.2 {% endif %} - ssh-import-id - locale @@ -102,7 +103,7 @@ cloud_config_modules: # The modules that run in the 'final' stage cloud_final_modules: {% if variant in ["ubuntu", "unknown", "debian"] %} - - snappy + - snappy # DEPRECATED- Drop in version 18.2 {% endif %} - package-update-upgrade-install {% if variant in ["ubuntu", "unknown", "debian"] %} diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 0ea3b6bf..50eb05cf 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -29,6 +29,7 @@ project = 'Cloud-Init' extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.autodoc', + 'sphinx.ext.autosectionlabel', 'sphinx.ext.viewcode', ] diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 7b146751..a0f68129 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -45,6 +45,7 @@ Modules .. automodule:: cloudinit.config.cc_seed_random .. automodule:: cloudinit.config.cc_set_hostname .. automodule:: cloudinit.config.cc_set_passwords +.. automodule:: cloudinit.config.cc_snap .. automodule:: cloudinit.config.cc_snappy .. automodule:: cloudinit.config.cc_snap_config .. automodule:: cloudinit.config.cc_spacewalk diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index d8bc170f..c7dcbe83 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -30,6 +30,9 @@ default_release_config: mirror_url: https://cloud-images.ubuntu.com/daily mirror_dir: '/srv/citest/images' keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg + # The OS version formatted as Major.Minor is used to compare releases + version: null # Each release needs to define this, for example 16.04 + ec2: # Choose from: [ebs, instance-store] root-store: ebs diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index 8e0fb62f..a3e29900 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -15,6 +15,9 @@ base_test_data: instance-id: | #!/bin/sh cat /run/cloud-init/.instance-id + instance-data.json: | + #!/bin/sh + cat /run/cloud-init/instance-data.json result.json: | #!/bin/sh cat /run/cloud-init/result.json diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index a29a0928..bd548f5a 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -7,6 +7,8 @@ import inspect import unittest from unittest.util import strclass +from cloudinit.util import read_conf + from tests.cloud_tests import config from tests.cloud_tests.testcases.base import CloudTestCase as base_test @@ -48,6 +50,7 @@ def get_suite(test_name, data, conf): def setUpClass(cls): cls.data = data cls.conf = conf + cls.release_conf = read_conf(config.RELEASES_CONF)['releases'] suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp)) diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 20e95955..324c7c91 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -4,10 +4,14 @@ import crypt import json +import re import unittest + from cloudinit import util as c_util +SkipTest = unittest.SkipTest + class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" @@ -16,6 +20,43 @@ class CloudTestCase(unittest.TestCase): data = {} conf = None _cloud_config = None + release_conf = {} # The platform's os release configuration + + expected_warnings = () # Subclasses set to ignore expected WARN logs + + @property + def os_cfg(self): + return self.release_conf[self.os_name]['default'] + + def is_distro(self, distro_name): + return self.os_cfg['os'] == distro_name + + def os_version_cmp(self, cmp_version): + """Compare the version of the test to comparison_version. + + @param: cmp_version: Either a float or a string representing + a release os from releases.yaml (e.g. centos66) + + @return: -1 when version < cmp_version, 0 when version=cmp_version and + 1 when version > cmp_version. + """ + version = self.release_conf[self.os_name]['default']['version'] + if isinstance(cmp_version, str): + cmp_version = self.release_conf[cmp_version]['default']['version'] + if version < cmp_version: + return -1 + elif version == cmp_version: + return 0 + else: + return 1 + + @property + def os_name(self): + return self.data.get('os_name', 'UNKNOWN') + + @property + def platform(self): + return self.data.get('platform', 'UNKNOWN') @property def cloud_config(self): @@ -72,12 +113,134 @@ class CloudTestCase(unittest.TestCase): self.assertEqual(len(result['errors']), 0) def test_no_warnings_in_log(self): - """Warnings should not be found in the log.""" + """Unexpected warnings should not be found in the log.""" + warnings = [ + l for l in self.get_data_file('cloud-init.log').splitlines() + if 'WARN' in l] + joined_warnings = '\n'.join(warnings) + for expected_warning in self.expected_warnings: + self.assertIn( + expected_warning, joined_warnings, + msg="Did not find %s in cloud-init.log" % expected_warning) + # Prune expected from discovered warnings + warnings = [w for w in warnings if expected_warning not in w] + self.assertEqual( + [], warnings, msg="'WARN' found inside cloud-init.log") + + def test_instance_data_json_ec2(self): + """Validate instance-data.json content by ec2 platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'ec2': + raise SkipTest( + 'Skipping ec2 instance-data.json on %s' % self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + self.assertEqual( + ['ds/user-data'], instance_data['base64-encoded-keys']) + ds = instance_data.get('ds', {}) + macs = ds.get('network', {}).get('interfaces', {}).get('macs', {}) + if not macs: + raise AssertionError('No network data from EC2 meta-data') + # Check meta-data items we depend on + expected_net_keys = [ + 'public-ipv4s', 'ipv4-associations', 'local-hostname', + 'public-hostname'] + for mac, mac_data in macs.items(): + for key in expected_net_keys: + self.assertIn(key, mac_data) + self.assertIsNotNone( + ds.get('placement', {}).get('availability-zone'), + 'Could not determine EC2 Availability zone placement') + ds = instance_data.get('ds', {}) + v1_data = instance_data.get('v1', {}) + self.assertIsNotNone( + v1_data['availability-zone'], 'expected ec2 availability-zone') + self.assertEqual('aws', v1_data['cloud-name']) + self.assertIn('i-', v1_data['instance-id']) + self.assertIn('ip-', v1_data['local-hostname']) + self.assertIsNotNone(v1_data['region'], 'expected ec2 region') + + def test_instance_data_json_lxd(self): + """Validate instance-data.json content by lxd platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'lxd': + raise SkipTest( + 'Skipping lxd instance-data.json on %s' % self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + v1_data = instance_data.get('v1', {}) + self.assertEqual( + ['ds/user-data', 'ds/vendor-data'], + sorted(instance_data['base64-encoded-keys'])) + self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertIsNone( + v1_data['availability-zone'], + 'found unexpected lxd availability-zone %s' % + v1_data['availability-zone']) + self.assertIn('cloud-test', v1_data['instance-id']) + self.assertIn('cloud-test', v1_data['local-hostname']) + self.assertIsNone( + v1_data['region'], + 'found unexpected lxd region %s' % v1_data['region']) + + def test_instance_data_json_kvm(self): + """Validate instance-data.json content by nocloud-kvm platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'nocloud-kvm': + raise SkipTest( + 'Skipping nocloud-kvm instance-data.json on %s' % + self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + v1_data = instance_data.get('v1', {}) self.assertEqual( - [], - [l for l in self.get_data_file('cloud-init.log').splitlines() - if 'WARN' in l], - msg="'WARN' found inside cloud-init.log") + ['ds/user-data'], instance_data['base64-encoded-keys']) + self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertIsNone( + v1_data['availability-zone'], + 'found unexpected kvm availability-zone %s' % + v1_data['availability-zone']) + self.assertIsNotNone( + re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', + v1_data['instance-id']), + 'kvm instance-id is not a UUID: %s' % v1_data['instance-id']) + self.assertIn('ubuntu', v1_data['local-hostname']) + self.assertIsNone( + v1_data['region'], + 'found unexpected lxd region %s' % v1_data['region']) class PasswordListTest(CloudTestCase): diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py index 857881cb..80a2c8d7 100644 --- a/tests/cloud_tests/testcases/main/command_output_simple.py +++ b/tests/cloud_tests/testcases/main/command_output_simple.py @@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base class TestCommandOutputSimple(base.CloudTestCase): """Test functionality of simple output redirection.""" + expected_warnings = ('Stdout, stderr changing to',) + def test_output_file(self): """Ensure that the output file is not empty and has all stages.""" data = self.get_data_file('cloud-init-test-output') @@ -15,20 +17,5 @@ class TestCommandOutputSimple(base.CloudTestCase): data.splitlines()[-1].strip()) # TODO: need to test that all stages redirected here - def test_no_warnings_in_log(self): - """Warnings should not be found in the log. - - This class redirected stderr and stdout, so it expects to find - a warning in cloud-init.log to that effect.""" - redirect_msg = 'Stdout, stderr changing to' - warnings = [ - l for l in self.get_data_file('cloud-init.log').splitlines() - if 'WARN' in l] - self.assertEqual( - [], [w for w in warnings if redirect_msg not in w], - msg="'WARN' found inside cloud-init.log") - self.assertEqual( - 1, len(warnings), - msg="Did not find %s in cloud-init.log" % redirect_msg) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snap.py b/tests/cloud_tests/testcases/modules/snap.py new file mode 100644 index 00000000..ff68abbe --- /dev/null +++ b/tests/cloud_tests/testcases/modules/snap.py @@ -0,0 +1,16 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script""" +from tests.cloud_tests.testcases import base + + +class TestSnap(base.CloudTestCase): + """Test snap module""" + + def test_snappy_version(self): + """Expect hello-world and core snaps are installed.""" + out = self.get_data_file('snaplist') + self.assertIn('core', out) + self.assertIn('hello-world', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snap.yaml b/tests/cloud_tests/testcases/modules/snap.yaml new file mode 100644 index 00000000..44043f31 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/snap.yaml @@ -0,0 +1,18 @@ +# +# Install snappy +# +required_features: + - snap +cloud_config: | + #cloud-config + package_update: true + snap: + squashfuse_in_container: true + commands: + - snap install hello-world +collect_scripts: + snaplist: | + #!/bin/bash + snap list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snappy.py b/tests/cloud_tests/testcases/modules/snappy.py index b92271c1..7d17fc5b 100644 --- a/tests/cloud_tests/testcases/modules/snappy.py +++ b/tests/cloud_tests/testcases/modules/snappy.py @@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base class TestSnappy(base.CloudTestCase): """Test snappy module""" + expected_warnings = ('DEPRECATION',) + def test_snappy_version(self): """Test snappy version output""" out = self.get_data_file('snapd') diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 2a9fd520..5a68a484 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -8,13 +8,16 @@ import unittest from tests.cloud_tests import (config, LOG, util, testcases) -def verify_data(base_dir, tests): +def verify_data(data_dir, platform, os_name, tests): """Verify test data is correct. - @param base_dir: base directory for data + @param data_dir: top level directory for all tests + @param platform: The platform name we for this test data (e.g. lxd) + @param os_name: The operating system under test (xenial, artful, etc.). @param tests: list of test names @return_value: {: {passed: True/False, failures: []}} """ + base_dir = os.sep.join((data_dir, platform, os_name)) runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: @@ -26,7 +29,7 @@ def verify_data(base_dir, tests): cloud_conf = test_conf['cloud_config'] # load script outputs - data = {} + data = {'platform': platform, 'os_name': os_name} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): with open(os.path.join(test_dir, script_name), 'rb') as fp: @@ -73,7 +76,7 @@ def verify(args): # run test res[platform][os_name] = verify_data( - os.sep.join((args.data_dir, platform, os_name)), + args.data_dir, platform, os_name, tests[platform][os_name]) # handle results diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 1ecb6c68..9b50ee79 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -26,6 +26,7 @@ class GetSchemaTest(CiTestCase): 'cc_ntp', 'cc_resizefs', 'cc_runcmd', + 'cc_snap', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 499e7c9f..67d9607d 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -785,6 +785,39 @@ class TestSubp(helpers.CiTestCase): decode=False) self.assertEqual(self.utf8_valid, out) + def test_bogus_command_logs_status_messages(self): + """status_cb gets status messages logs on bogus commands provided.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(util.ProcessExecutionError): + util.subp([self.bogus_command], status_cb=status_cb) + + expected = [ + 'Begin run command: {cmd}\n'.format(cmd=self.bogus_command), + 'ERROR: End run command: invalid command provided\n'] + self.assertEqual(expected, logs) + + def test_command_logs_exit_codes_to_status_cb(self): + """status_cb gets status messages containing command exit code.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(util.ProcessExecutionError): + util.subp(['ls', '/I/dont/exist'], status_cb=status_cb) + util.subp(['ls'], status_cb=status_cb) + + expected = [ + 'Begin run command: ls /I/dont/exist\n', + 'ERROR: End run command: exit(2)\n', + 'Begin run command: ls\n', + 'End run command: exit(0)\n'] + self.assertEqual(expected, logs) + class TestEncode(helpers.TestCase): """Test the encoding functions""" -- cgit v1.2.3 From de34dc7c467b318b2d04d065f8d752c7a530e155 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Mar 2018 15:43:27 -0600 Subject: net: recognize iscsi root cases without ip= on kernel command line. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When 'ip=' or 'ip6=' is found on the kernel command line, cloud-init will consider read network config from /run/net-*.conf files. There are some iscsi-root scenarios where initramfs configures networking but the ip= parameter is not present. 2 such cases are:  a.) static config in /etc/iscsi/iscsi.initramfs (copied into the initramfs)  b.) iBft This changes cloud-init to consider initramfs provided networking information if:  * there are /run/net-* files and  * (ip= or ip6 is on the command line) or open-iscsi.interface file exists. LP: #1752391 --- cloudinit/net/cmdline.py | 24 ++++++++++++++++-- cloudinit/tests/helpers.py | 9 +++++-- tests/unittests/test_net.py | 61 ++++++++++++++++++++++++++++++++------------- 3 files changed, 72 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 7b2cc9db..9e9fe0fe 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -9,12 +9,15 @@ import base64 import glob import gzip import io +import os from . import get_devicelist from . import read_sys_net_safe from cloudinit import util +_OPEN_ISCSI_INTERFACE_FILE = "/run/initramfs/open-iscsi.interface" + def _klibc_to_config_entry(content, mac_addrs=None): """Convert a klibc written shell content file to a 'config' entry @@ -103,9 +106,13 @@ def _klibc_to_config_entry(content, mac_addrs=None): return name, iface +def _get_klibc_net_cfg_files(): + return glob.glob('/run/net-*.conf') + glob.glob('/run/net6-*.conf') + + def config_from_klibc_net_cfg(files=None, mac_addrs=None): if files is None: - files = glob.glob('/run/net-*.conf') + glob.glob('/run/net6-*.conf') + files = _get_klibc_net_cfg_files() entries = [] names = {} @@ -160,10 +167,23 @@ def _b64dgz(b64str, gzipped="try"): return _decomp_gzip(blob, strict=gzipped != "try") +def _is_initramfs_netconfig(files, cmdline): + if files: + if 'ip=' in cmdline or 'ip6=' in cmdline: + return True + if os.path.exists(_OPEN_ISCSI_INTERFACE_FILE): + # iBft can configure networking without ip= + return True + return False + + def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if cmdline is None: cmdline = util.get_cmdline() + if files is None: + files = _get_klibc_net_cfg_files() + if 'network-config=' in cmdline: data64 = None for tok in cmdline.split(): @@ -172,7 +192,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if data64: return util.load_yaml(_b64dgz(data64)) - if 'ip=' not in cmdline and 'ip6=' not in cmdline: + if not _is_initramfs_netconfig(files, cmdline): return None if mac_addrs is None: diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index a2e10536..999b1d7c 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -283,10 +283,15 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def patchOS(self, new_root): patch_funcs = { os.path: [('isfile', 1), ('exists', 1), - ('islink', 1), ('isdir', 1)], + ('islink', 1), ('isdir', 1), ('lexists', 1)], os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)], + ('lstat', 1), ('symlink', 2)] } + + if hasattr(os, 'scandir'): + # py27 does not have scandir + patch_funcs[os].append(('scandir', 1)) + for (mod, funcs) in patch_funcs.items(): for f, nargs in funcs: func = getattr(mod, f) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 9cf11f2e..84a0eabf 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -12,10 +12,8 @@ from cloudinit.sources.helpers import openstack from cloudinit import temp_utils from cloudinit import util -from cloudinit.tests.helpers import CiTestCase -from cloudinit.tests.helpers import dir2dict -from cloudinit.tests.helpers import mock -from cloudinit.tests.helpers import populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, FilesystemMockingTestCase, dir2dict, mock, populate_dir) import base64 import copy @@ -2186,27 +2184,49 @@ class TestCmdlineConfigParsing(CiTestCase): self.assertEqual(found, self.simple_cfg) -class TestCmdlineReadKernelConfig(CiTestCase): +class TestCmdlineReadKernelConfig(FilesystemMockingTestCase): macs = { 'eth0': '14:02:ec:42:48:00', 'eno1': '14:02:ec:42:48:01', } - def test_ip_cmdline_read_kernel_cmdline_ip(self): - content = {'net-eth0.conf': DHCP_CONTENT_1} - files = sorted(populate_dir(self.tmp_dir(), content)) + def test_ip_cmdline_without_ip(self): + content = {'/run/net-eth0.conf': DHCP_CONTENT_1, + cmdline._OPEN_ISCSI_INTERFACE_FILE: "eth0\n"} + exp1 = copy.deepcopy(DHCP_EXPECTED_1) + exp1['mac_address'] = self.macs['eth0'] + + root = self.tmp_dir() + populate_dir(root, content) + self.reRoot(root) + found = cmdline.read_kernel_cmdline_config( - files=files, cmdline='foo ip=dhcp', mac_addrs=self.macs) + cmdline='foo root=/root/bar', mac_addrs=self.macs) + self.assertEqual(found['version'], 1) + self.assertEqual(found['config'], [exp1]) + + def test_ip_cmdline_read_kernel_cmdline_ip(self): + content = {'/run/net-eth0.conf': DHCP_CONTENT_1} exp1 = copy.deepcopy(DHCP_EXPECTED_1) exp1['mac_address'] = self.macs['eth0'] + + root = self.tmp_dir() + populate_dir(root, content) + self.reRoot(root) + + found = cmdline.read_kernel_cmdline_config( + cmdline='foo ip=dhcp', mac_addrs=self.macs) self.assertEqual(found['version'], 1) self.assertEqual(found['config'], [exp1]) def test_ip_cmdline_read_kernel_cmdline_ip6(self): - content = {'net6-eno1.conf': DHCP6_CONTENT_1} - files = sorted(populate_dir(self.tmp_dir(), content)) + content = {'/run/net6-eno1.conf': DHCP6_CONTENT_1} + root = self.tmp_dir() + populate_dir(root, content) + self.reRoot(root) + found = cmdline.read_kernel_cmdline_config( - files=files, cmdline='foo ip6=dhcp root=/dev/sda', + cmdline='foo ip6=dhcp root=/dev/sda', mac_addrs=self.macs) self.assertEqual( found, @@ -2226,18 +2246,23 @@ class TestCmdlineReadKernelConfig(CiTestCase): self.assertIsNone(found) def test_ip_cmdline_both_ip_ip6(self): - content = {'net-eth0.conf': DHCP_CONTENT_1, - 'net6-eth0.conf': DHCP6_CONTENT_1.replace('eno1', 'eth0')} - files = sorted(populate_dir(self.tmp_dir(), content)) - found = cmdline.read_kernel_cmdline_config( - files=files, cmdline='foo ip=dhcp ip6=dhcp', mac_addrs=self.macs) - + content = { + '/run/net-eth0.conf': DHCP_CONTENT_1, + '/run/net6-eth0.conf': DHCP6_CONTENT_1.replace('eno1', 'eth0')} eth0 = copy.deepcopy(DHCP_EXPECTED_1) eth0['mac_address'] = self.macs['eth0'] eth0['subnets'].append( {'control': 'manual', 'type': 'dhcp6', 'netmask': '64', 'dns_nameservers': ['2001:67c:1562:8010::2:1']}) expected = [eth0] + + root = self.tmp_dir() + populate_dir(root, content) + self.reRoot(root) + + found = cmdline.read_kernel_cmdline_config( + cmdline='foo ip=dhcp ip6=dhcp', mac_addrs=self.macs) + self.assertEqual(found['version'], 1) self.assertEqual(found['config'], expected) -- cgit v1.2.3 From e88e35483e373b39b4485f30f7a867f50571027c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 19 Mar 2018 14:50:54 -0400 Subject: Add Hetzner Cloud DataSource The Hetzner Cloud metadata service is an AWS-style service available over HTTP via the link local address 169.254.169.254. https://hetzner.com/cloud https://docs.hetzner.cloud/ --- cloudinit/apport.py | 6 +- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceHetzner.py | 100 ++++++++++++++++++++++++ cloudinit/sources/helpers/hetzner.py | 26 ++++++ tests/unittests/test_datasource/test_common.py | 2 + tests/unittests/test_datasource/test_hetzner.py | 99 +++++++++++++++++++++++ tests/unittests/test_ds_identify.py | 9 +++ tools/ds-identify | 7 +- 8 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 cloudinit/sources/DataSourceHetzner.py create mode 100644 cloudinit/sources/helpers/hetzner.py create mode 100644 tests/unittests/test_datasource/test_hetzner.py (limited to 'tests') diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 221f341c..618b0160 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -14,9 +14,9 @@ except ImportError: KNOWN_CLOUD_NAMES = [ 'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma', - 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS', - 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS', - 'VMware', 'Other'] + 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', + 'Hetzner Cloud', 'MAAS', 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', + 'Scaleway', 'SmartOS', 'VMware', 'Other'] # Potentially clear text collected logs CLOUDINIT_LOG = '/var/log/cloud-init.log' diff --git a/cloudinit/settings.py b/cloudinit/settings.py index c120498f..5fe749d4 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -36,6 +36,7 @@ CFG_BUILTIN = { 'SmartOS', 'Bigstep', 'Scaleway', + 'Hetzner', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py new file mode 100644 index 00000000..769fe131 --- /dev/null +++ b/cloudinit/sources/DataSourceHetzner.py @@ -0,0 +1,100 @@ +# Author: Jonas Keidel +# Author: Markus Schade +# +# This file is part of cloud-init. See LICENSE file for license information. +# +"""Hetzner Cloud API Documentation. + https://docs.hetzner.cloud/""" + +from cloudinit import log as logging +from cloudinit import net as cloudnet +from cloudinit import sources +from cloudinit import util + +import cloudinit.sources.helpers.hetzner as hc_helper + +LOG = logging.getLogger(__name__) + +BASE_URL_V1 = 'http://169.254.169.254/hetzner/v1' + +BUILTIN_DS_CONFIG = { + 'metadata_url': BASE_URL_V1 + '/metadata', + 'userdata_url': BASE_URL_V1 + '/userdata', +} + +MD_RETRIES = 60 +MD_TIMEOUT = 2 +MD_WAIT_RETRY = 2 + + +class DataSourceHetzner(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.distro = distro + self.metadata = dict() + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "Hetzner"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] + self.userdata_address = self.ds_cfg['userdata_url'] + self.retries = self.ds_cfg.get('retries', MD_RETRIES) + self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT) + self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY) + self._network_config = None + self.dsmode = sources.DSMODE_NETWORK + + def get_data(self): + nic = cloudnet.find_fallback_nic() + with cloudnet.EphemeralIPv4Network(nic, "169.254.0.1", 16, + "169.254.255.255"): + md = hc_helper.read_metadata( + self.metadata_address, timeout=self.timeout, + sec_between=self.wait_retry, retries=self.retries) + ud = hc_helper.read_userdata( + self.userdata_address, timeout=self.timeout, + sec_between=self.wait_retry, retries=self.retries) + + self.userdata_raw = ud + self.metadata_full = md + + """hostname is name provided by user at launch. The API enforces + it is a valid hostname, but it is not guaranteed to be resolvable + in dns or fully qualified.""" + self.metadata['instance-id'] = md['instance-id'] + self.metadata['local-hostname'] = md['hostname'] + self.metadata['network-config'] = md.get('network-config', None) + self.metadata['public-keys'] = md.get('public-keys', None) + self.vendordata_raw = md.get("vendor_data", None) + + return True + + @property + def network_config(self): + """Configure the networking. This needs to be done each boot, since + the IP information may have changed due to snapshot and/or + migration. + """ + + if self._network_config: + return self._network_config + + _net_config = self.metadata['network-config'] + if not _net_config: + raise Exception("Unable to get meta-data from server....") + + self._network_config = _net_config + + return self._network_config + + +# Used to match classes to dependencies +datasources = [ + (DataSourceHetzner, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py new file mode 100644 index 00000000..2554530d --- /dev/null +++ b/cloudinit/sources/helpers/hetzner.py @@ -0,0 +1,26 @@ +# Author: Jonas Keidel +# Author: Markus Schade +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import log as logging +from cloudinit import url_helper +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +def read_metadata(url, timeout=2, sec_between=2, retries=30): + response = url_helper.readurl(url, timeout=timeout, + sec_between=sec_between, retries=retries) + if not response.ok(): + raise RuntimeError("unable to read metadata at %s" % url) + return util.load_yaml(response.contents.decode()) + + +def read_userdata(url, timeout=2, sec_between=2, retries=30): + response = url_helper.readurl(url, timeout=timeout, + sec_between=sec_between, retries=retries) + if not response.ok(): + raise RuntimeError("unable to read userdata at %s" % url) + return response.contents diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 80b9c650..6d2dc5b5 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -14,6 +14,7 @@ from cloudinit.sources import ( DataSourceDigitalOcean as DigitalOcean, DataSourceEc2 as Ec2, DataSourceGCE as GCE, + DataSourceHetzner as Hetzner, DataSourceMAAS as MAAS, DataSourceNoCloud as NoCloud, DataSourceOpenNebula as OpenNebula, @@ -31,6 +32,7 @@ DEFAULT_LOCAL = [ CloudSigma.DataSourceCloudSigma, ConfigDrive.DataSourceConfigDrive, DigitalOcean.DataSourceDigitalOcean, + Hetzner.DataSourceHetzner, NoCloud.DataSourceNoCloud, OpenNebula.DataSourceOpenNebula, OVF.DataSourceOVF, diff --git a/tests/unittests/test_datasource/test_hetzner.py b/tests/unittests/test_datasource/test_hetzner.py new file mode 100644 index 00000000..f1d1525e --- /dev/null +++ b/tests/unittests/test_datasource/test_hetzner.py @@ -0,0 +1,99 @@ +# Copyright (C) 2018 Jonas Keidel +# +# Author: Jonas Keidel +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.sources import DataSourceHetzner +from cloudinit import util, settings, helpers + +from cloudinit.tests.helpers import mock, CiTestCase + +METADATA = util.load_yaml(""" +hostname: cloudinit-test +instance-id: 123456 +local-ipv4: '' +network-config: + config: + - mac_address: 96:00:00:08:19:da + name: eth0 + subnets: + - dns_nameservers: + - 213.133.99.99 + - 213.133.100.100 + - 213.133.98.98 + ipv4: true + type: dhcp + type: physical + - name: eth0:0 + subnets: + - address: 2a01:4f8:beef:beef::1/64 + gateway: fe80::1 + ipv6: true + routes: + - gateway: fe80::1%eth0 + netmask: 0 + network: '::' + type: static + type: physical + version: 1 +network-sysconfig: "DEVICE='eth0'\nTYPE=Ethernet\nBOOTPROTO=dhcp\n\ + ONBOOT='yes'\nHWADDR=96:00:00:08:19:da\n\ + IPV6INIT=yes\nIPV6ADDR=2a01:4f8:beef:beef::1/64\n\ + IPV6_DEFAULTGW=fe80::1%eth0\nIPV6_AUTOCONF=no\n\ + DNS1=213.133.99.99\nDNS2=213.133.100.100\n" +public-ipv4: 192.168.0.1 +public-keys: +- ssh-ed25519 \ + AAAAC3Nzac1lZdI1NTE5AaaAIaFrcac0yVITsmRrmueq6MD0qYNKlEvW8O1Ib4nkhmWh \ + test-key@workstation +vendor_data: "test" +""") + +USERDATA = b"""#cloud-config +runcmd: +- [touch, /root/cloud-init-worked ] +""" + + +class TestDataSourceHetzner(CiTestCase): + """ + Test reading the meta-data + """ + def setUp(self): + super(TestDataSourceHetzner, self).setUp() + self.tmp = self.tmp_dir() + + def get_ds(self): + ds = DataSourceHetzner.DataSourceHetzner( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + return ds + + @mock.patch('cloudinit.net.EphemeralIPv4Network') + @mock.patch('cloudinit.net.find_fallback_nic') + @mock.patch('cloudinit.sources.helpers.hetzner.read_metadata') + @mock.patch('cloudinit.sources.helpers.hetzner.read_userdata') + def test_read_data(self, m_usermd, m_readmd, m_fallback_nic, m_net): + m_readmd.return_value = METADATA.copy() + m_usermd.return_value = USERDATA + m_fallback_nic.return_value = 'eth0' + + ds = self.get_ds() + ret = ds.get_data() + self.assertTrue(ret) + + m_net.assert_called_once_with( + 'eth0', '169.254.0.1', + 16, '169.254.255.255' + ) + + self.assertTrue(m_readmd.called) + + self.assertEqual(METADATA.get('hostname'), ds.get_hostname()) + + self.assertEqual(METADATA.get('public-keys'), + ds.get_public_ssh_keys()) + + self.assertIsInstance(ds.get_public_ssh_keys(), list) + self.assertEqual(ds.get_userdata_raw(), USERDATA) + self.assertEqual(ds.get_vendordata_raw(), METADATA.get('vendor_data')) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 9be3f964..9c5628e7 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -60,6 +60,7 @@ P_CHASSIS_ASSET_TAG = "sys/class/dmi/id/chassis_asset_tag" P_PRODUCT_NAME = "sys/class/dmi/id/product_name" P_PRODUCT_SERIAL = "sys/class/dmi/id/product_serial" P_PRODUCT_UUID = "sys/class/dmi/id/product_uuid" +P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor" P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" @@ -379,6 +380,10 @@ class TestDsIdentify(CiTestCase): """Nocloud seed directory ubuntu core writable""" self._test_ds_found('NoCloud-seed-ubuntu-core') + def test_hetzner_found(self): + """Hetzner cloud is identified in sys_vendor.""" + self._test_ds_found('Hetzner') + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" @@ -559,6 +564,10 @@ VALID_CFG = { }, ], }, + 'Hetzner': { + 'ds': 'Hetzner', + 'files': {P_SYS_VENDOR: 'Hetzner\n'}, + }, } # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index ec368d58..e3f93c90 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -114,7 +114,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway" +OVF SmartOS Scaleway Hetzner" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -979,6 +979,11 @@ dscheck_Scaleway() { return ${DS_NOT_FOUND} } +dscheck_Hetzner() { + dmi_sys_vendor_is Hetzner && return ${DS_FOUND} + return ${DS_NOT_FOUND} +} + collect_info() { read_virt read_pid1_product_name -- cgit v1.2.3 From 7deec7b6a1fce87dc2d9cd886053804bbc70380e Mon Sep 17 00:00:00 2001 From: Akihiko Ota Date: Tue, 20 Mar 2018 11:41:26 -0600 Subject: OpenNebula: Update network to return v2 config rather than ENI. OpenNebulaNetwork.gen_conf() was previously returning ENI format. This is updated to return netplan/v2 config. The changes here also adds support for IPv6 configuration distributed from OpenNebula and fixes some issues about nameserver information. --- cloudinit/net/network_state.py | 10 + cloudinit/sources/DataSourceOpenNebula.py | 104 +++++--- tests/unittests/test_datasource/test_opennebula.py | 266 ++++++++++++++------- 3 files changed, 261 insertions(+), 119 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 1dd7ded7..6d63e5c5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -708,6 +708,7 @@ class NetworkStateInterpreter(object): gateway4 = None gateway6 = None + nameservers = {} for address in cfg.get('addresses', []): subnet = { 'type': 'static', @@ -723,6 +724,15 @@ class NetworkStateInterpreter(object): gateway4 = cfg.get('gateway4') subnet.update({'gateway': gateway4}) + if 'nameservers' in cfg and not nameservers: + addresses = cfg.get('nameservers').get('addresses') + if addresses: + nameservers['dns_nameservers'] = addresses + search = cfg.get('nameservers').get('search') + if search: + nameservers['dns_search'] = search + subnet.update(nameservers) + subnets.append(subnet) routes = [] diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 9450835e..02cb98f7 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -20,7 +20,6 @@ import string from cloudinit import log as logging from cloudinit import net -from cloudinit.net import eni from cloudinit import sources from cloudinit import util @@ -91,15 +90,15 @@ class DataSourceOpenNebula(sources.DataSource): return False self.seed = seed - self.network_eni = results.get('network-interfaces') + self.network = results.get('network-interfaces') self.metadata = md self.userdata_raw = results.get('userdata') return True @property def network_config(self): - if self.network_eni is not None: - return eni.convert_eni_data(self.network_eni) + if self.network is not None: + return self.network else: return None @@ -143,18 +142,42 @@ class OpenNebulaNetwork(object): def mac2network(self, mac): return self.mac2ip(mac).rpartition(".")[0] + ".0" - def get_dns(self, dev): - return self.get_field(dev, "dns", "").split() + def get_nameservers(self, dev): + nameservers = {} + dns = self.get_field(dev, "dns", "").split() + dns.extend(self.context.get('DNS', "").split()) + if dns: + nameservers['addresses'] = dns + search_domain = self.get_field(dev, "search_domain", "").split() + if search_domain: + nameservers['search'] = search_domain + return nameservers - def get_domain(self, dev): - return self.get_field(dev, "domain") + def get_mtu(self, dev): + return self.get_field(dev, "mtu") def get_ip(self, dev, mac): return self.get_field(dev, "ip", self.mac2ip(mac)) + def get_ip6(self, dev): + addresses6 = [] + ip6 = self.get_field(dev, "ip6") + if ip6: + addresses6.append(ip6) + ip6_ula = self.get_field(dev, "ip6_ula") + if ip6_ula: + addresses6.append(ip6_ula) + return addresses6 + + def get_ip6_prefix(self, dev): + return self.get_field(dev, "ip6_prefix_length", "64") + def get_gateway(self, dev): return self.get_field(dev, "gateway") + def get_gateway6(self, dev): + return self.get_field(dev, "gateway6") + def get_mask(self, dev): return self.get_field(dev, "mask", "255.255.255.0") @@ -171,10 +194,11 @@ class OpenNebulaNetwork(object): return default if val in (None, "") else val def gen_conf(self): - global_dns = self.context.get('DNS', "").split() - - conf = ['auto lo', 'iface lo inet loopback', ''] + netconf = {} + netconf['version'] = 2 + netconf['ethernets'] = {} + ethernets = {} for mac, dev in self.ifaces.items(): mac = mac.lower() @@ -182,29 +206,49 @@ class OpenNebulaNetwork(object): # dev stores the current system name. c_dev = self.context_devname.get(mac, dev) - conf.append('auto ' + dev) - conf.append('iface ' + dev + ' inet static') - conf.append(' #hwaddress %s' % mac) - conf.append(' address ' + self.get_ip(c_dev, mac)) - conf.append(' network ' + self.get_network(c_dev, mac)) - conf.append(' netmask ' + self.get_mask(c_dev)) + devconf = {} + + # Set MAC address + devconf['match'] = {'macaddress': mac} + # Set IPv4 address + devconf['addresses'] = [] + mask = self.get_mask(c_dev) + prefix = str(net.mask_to_net_prefix(mask)) + devconf['addresses'].append( + self.get_ip(c_dev, mac) + '/' + prefix) + + # Set IPv6 Global and ULA address + addresses6 = self.get_ip6(c_dev) + if addresses6: + prefix6 = self.get_ip6_prefix(c_dev) + devconf['addresses'].extend( + [i + '/' + prefix6 for i in addresses6]) + + # Set IPv4 default gateway gateway = self.get_gateway(c_dev) if gateway: - conf.append(' gateway ' + gateway) + devconf['gateway4'] = gateway + + # Set IPv6 default gateway + gateway6 = self.get_gateway6(c_dev) + if gateway: + devconf['gateway6'] = gateway6 - domain = self.get_domain(c_dev) - if domain: - conf.append(' dns-search ' + domain) + # Set DNS servers and search domains + nameservers = self.get_nameservers(c_dev) + if nameservers: + devconf['nameservers'] = nameservers - # add global DNS servers to all interfaces - dns = self.get_dns(c_dev) - if global_dns or dns: - conf.append(' dns-nameservers ' + ' '.join(global_dns + dns)) + # Set MTU size + mtu = self.get_mtu(c_dev) + if mtu: + devconf['mtu'] = mtu - conf.append('') + ethernets[dev] = devconf - return "\n".join(conf) + netconf['ethernets'] = ethernets + return(netconf) def find_candidate_devs(): @@ -390,10 +434,10 @@ def read_context_disk_dir(source_dir, asuser=None): except TypeError: LOG.warning("Failed base64 decoding of userdata") - # generate static /etc/network/interfaces + # generate Network Configuration v2 # only if there are any required context variables - # http://opennebula.org/documentation:rel3.8:cong#network_configuration - ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)] + # http://docs.opennebula.org/5.4/operation/references/template.html#context-section + ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP.*$', k)] if ipaddr_keys: onet = OpenNebulaNetwork(context) results['network-interfaces'] = onet.gen_conf() diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 5c3ba012..ab42f344 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -4,7 +4,6 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from cloudinit.tests.helpers import mock, populate_dir, CiTestCase -from textwrap import dedent import os import pwd @@ -33,6 +32,11 @@ HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' MACADDR = '02:00:0a:12:01:01' IP_BY_MACADDR = '10.18.1.1' +IP4_PREFIX = '24' +IP6_GLOBAL = '2001:db8:1:0:400:c0ff:fea8:1ba' +IP6_ULA = 'fd01:dead:beaf:0:400:c0ff:fea8:1ba' +IP6_GW = '2001:db8:1::ffff' +IP6_PREFIX = '48' DS_PATH = "cloudinit.sources.DataSourceOpenNebula" @@ -221,7 +225,9 @@ class TestOpenNebulaDataSource(CiTestCase): results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue(IP_BY_MACADDR in results['network-interfaces']) + self.assertTrue( + IP_BY_MACADDR + '/' + IP4_PREFIX in + results['network-interfaces']['ethernets'][dev]['addresses']) # ETH0_IP and ETH0_MAC populate_context_dir( @@ -229,7 +235,9 @@ class TestOpenNebulaDataSource(CiTestCase): results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue(IP_BY_MACADDR in results['network-interfaces']) + self.assertTrue( + IP_BY_MACADDR + '/' + IP4_PREFIX in + results['network-interfaces']['ethernets'][dev]['addresses']) # ETH0_IP with empty string and ETH0_MAC # in the case of using Virtual Network contains @@ -239,55 +247,91 @@ class TestOpenNebulaDataSource(CiTestCase): results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue(IP_BY_MACADDR in results['network-interfaces']) + self.assertTrue( + IP_BY_MACADDR + '/' + IP4_PREFIX in + results['network-interfaces']['ethernets'][dev]['addresses']) - # ETH0_NETWORK + # ETH0_MASK populate_context_dir( self.seed_dir, { 'ETH0_IP': IP_BY_MACADDR, 'ETH0_MAC': MACADDR, - 'ETH0_NETWORK': '10.18.0.0' + 'ETH0_MASK': '255.255.0.0' }) results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue('10.18.0.0' in results['network-interfaces']) + self.assertTrue( + IP_BY_MACADDR + '/16' in + results['network-interfaces']['ethernets'][dev]['addresses']) - # ETH0_NETWORK with empty string + # ETH0_MASK with empty string populate_context_dir( self.seed_dir, { 'ETH0_IP': IP_BY_MACADDR, 'ETH0_MAC': MACADDR, - 'ETH0_NETWORK': '' + 'ETH0_MASK': '' }) results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue('10.18.1.0' in results['network-interfaces']) + self.assertTrue( + IP_BY_MACADDR + '/' + IP4_PREFIX in + results['network-interfaces']['ethernets'][dev]['addresses']) - # ETH0_MASK + # ETH0_IP6 populate_context_dir( self.seed_dir, { - 'ETH0_IP': IP_BY_MACADDR, + 'ETH0_IP6': IP6_GLOBAL, 'ETH0_MAC': MACADDR, - 'ETH0_MASK': '255.255.0.0' }) results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue('255.255.0.0' in results['network-interfaces']) + self.assertTrue( + IP6_GLOBAL + '/64' in + results['network-interfaces']['ethernets'][dev]['addresses']) - # ETH0_MASK with empty string + # ETH0_IP6_ULA populate_context_dir( self.seed_dir, { - 'ETH0_IP': IP_BY_MACADDR, + 'ETH0_IP6_ULA': IP6_ULA, + 'ETH0_MAC': MACADDR, + }) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertTrue('network-interfaces' in results) + self.assertTrue( + IP6_ULA + '/64' in + results['network-interfaces']['ethernets'][dev]['addresses']) + + # ETH0_IP6 and ETH0_IP6_PREFIX_LENGTH + populate_context_dir( + self.seed_dir, { + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX, + 'ETH0_MAC': MACADDR, + }) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertTrue('network-interfaces' in results) + self.assertTrue( + IP6_GLOBAL + '/' + IP6_PREFIX in + results['network-interfaces']['ethernets'][dev]['addresses']) + + # ETH0_IP6 and ETH0_IP6_PREFIX_LENGTH with empty string + populate_context_dir( + self.seed_dir, { + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_PREFIX_LENGTH': '', 'ETH0_MAC': MACADDR, - 'ETH0_MASK': '' }) results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) - self.assertTrue('255.255.255.0' in results['network-interfaces']) + self.assertTrue( + IP6_GLOBAL + '/64' in + results['network-interfaces']['ethernets'][dev]['addresses']) def test_find_candidates(self): def my_devs_with(criteria): @@ -310,108 +354,152 @@ class TestOpenNebulaNetwork(unittest.TestCase): system_nics = ('eth0', 'ens3') - def test_lo(self): - net = ds.OpenNebulaNetwork(context={}, system_nics_by_mac={}) - self.assertEqual(net.gen_conf(), u'''\ -auto lo -iface lo inet loopback -''') - @mock.patch(DS_PATH + ".get_physical_nics_by_mac") def test_eth0(self, m_get_phys_by_mac): for nic in self.system_nics: m_get_phys_by_mac.return_value = {MACADDR: nic} net = ds.OpenNebulaNetwork({}) - self.assertEqual(net.gen_conf(), dedent("""\ - auto lo - iface lo inet loopback - - auto {dev} - iface {dev} inet static - #hwaddress {macaddr} - address 10.18.1.1 - network 10.18.1.0 - netmask 255.255.255.0 - """.format(dev=nic, macaddr=MACADDR))) + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + + self.assertEqual(net.gen_conf(), expected) def test_eth0_override(self): + self.maxDiff = None context = { 'DNS': '1.2.3.8', - 'ETH0_IP': '10.18.1.1', - 'ETH0_NETWORK': '10.18.0.0', - 'ETH0_MASK': '255.255.0.0', + 'ETH0_DNS': '1.2.3.6 1.2.3.7', 'ETH0_GATEWAY': '1.2.3.5', - 'ETH0_DOMAIN': 'example.com', + 'ETH0_GATEWAY6': '', + 'ETH0_IP': IP_BY_MACADDR, + 'ETH0_IP6': '', + 'ETH0_IP6_PREFIX_LENGTH': '', + 'ETH0_IP6_ULA': '', + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_MASK': '255.255.0.0', + 'ETH0_MTU': '', + 'ETH0_NETWORK': '10.18.0.0', + 'ETH0_SEARCH_DOMAIN': '', + } + for nic in self.system_nics: + net = ds.OpenNebulaNetwork(context, + system_nics_by_mac={MACADDR: nic}) + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/16'], + 'gateway4': '1.2.3.5', + 'gateway6': None, + 'nameservers': { + 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8']}}}} + + self.assertEqual(expected, net.gen_conf()) + + def test_eth0_v4v6_override(self): + self.maxDiff = None + context = { + 'DNS': '1.2.3.8', 'ETH0_DNS': '1.2.3.6 1.2.3.7', - 'ETH0_MAC': '02:00:0a:12:01:01' + 'ETH0_GATEWAY': '1.2.3.5', + 'ETH0_GATEWAY6': IP6_GW, + 'ETH0_IP': IP_BY_MACADDR, + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX, + 'ETH0_IP6_ULA': IP6_ULA, + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_MASK': '255.255.0.0', + 'ETH0_MTU': '1280', + 'ETH0_NETWORK': '10.18.0.0', + 'ETH0_SEARCH_DOMAIN': 'example.com example.org', } for nic in self.system_nics: - expected = dedent("""\ - auto lo - iface lo inet loopback - - auto {dev} - iface {dev} inet static - #hwaddress {macaddr} - address 10.18.1.1 - network 10.18.0.0 - netmask 255.255.0.0 - gateway 1.2.3.5 - dns-search example.com - dns-nameservers 1.2.3.8 1.2.3.6 1.2.3.7 - """).format(dev=nic, macaddr=MACADDR) net = ds.OpenNebulaNetwork(context, system_nics_by_mac={MACADDR: nic}) + + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [ + IP_BY_MACADDR + '/16', + IP6_GLOBAL + '/' + IP6_PREFIX, + IP6_ULA + '/' + IP6_PREFIX], + 'gateway4': '1.2.3.5', + 'gateway6': IP6_GW, + 'nameservers': { + 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'], + 'search': ['example.com', 'example.org']}, + 'mtu': '1280'}}} + self.assertEqual(expected, net.gen_conf()) def test_multiple_nics(self): """Test rendering multiple nics with names that differ from context.""" + self.maxDiff = None MAC_1 = "02:00:0a:12:01:01" MAC_2 = "02:00:0a:12:01:02" context = { 'DNS': '1.2.3.8', - 'ETH0_IP': '10.18.1.1', - 'ETH0_NETWORK': '10.18.0.0', - 'ETH0_MASK': '255.255.0.0', - 'ETH0_GATEWAY': '1.2.3.5', - 'ETH0_DOMAIN': 'example.com', 'ETH0_DNS': '1.2.3.6 1.2.3.7', + 'ETH0_GATEWAY': '1.2.3.5', + 'ETH0_GATEWAY6': IP6_GW, + 'ETH0_IP': '10.18.1.1', + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_PREFIX_LENGTH': '', + 'ETH0_IP6_ULA': IP6_ULA, 'ETH0_MAC': MAC_2, - 'ETH3_IP': '10.3.1.3', - 'ETH3_NETWORK': '10.3.0.0', - 'ETH3_MASK': '255.255.0.0', - 'ETH3_GATEWAY': '10.3.0.1', - 'ETH3_DOMAIN': 'third.example.com', + 'ETH0_MASK': '255.255.0.0', + 'ETH0_MTU': '1280', + 'ETH0_NETWORK': '10.18.0.0', + 'ETH0_SEARCH_DOMAIN': 'example.com', 'ETH3_DNS': '10.3.1.2', + 'ETH3_GATEWAY': '10.3.0.1', + 'ETH3_GATEWAY6': '', + 'ETH3_IP': '10.3.1.3', + 'ETH3_IP6': '', + 'ETH3_IP6_PREFIX_LENGTH': '', + 'ETH3_IP6_ULA': '', 'ETH3_MAC': MAC_1, + 'ETH3_MASK': '255.255.0.0', + 'ETH3_MTU': '', + 'ETH3_NETWORK': '10.3.0.0', + 'ETH3_SEARCH_DOMAIN': 'third.example.com third.example.org', } net = ds.OpenNebulaNetwork( context, system_nics_by_mac={MAC_1: 'enp0s25', MAC_2: 'enp1s2'}) - expected = dedent("""\ - auto lo - iface lo inet loopback - - auto enp0s25 - iface enp0s25 inet static - #hwaddress 02:00:0a:12:01:01 - address 10.3.1.3 - network 10.3.0.0 - netmask 255.255.0.0 - gateway 10.3.0.1 - dns-search third.example.com - dns-nameservers 1.2.3.8 10.3.1.2 - - auto enp1s2 - iface enp1s2 inet static - #hwaddress 02:00:0a:12:01:02 - address 10.18.1.1 - network 10.18.0.0 - netmask 255.255.0.0 - gateway 1.2.3.5 - dns-search example.com - dns-nameservers 1.2.3.8 1.2.3.6 1.2.3.7 - """) + expected = { + 'version': 2, + 'ethernets': { + 'enp1s2': { + 'match': {'macaddress': MAC_2}, + 'addresses': [ + '10.18.1.1/16', + IP6_GLOBAL + '/64', + IP6_ULA + '/64'], + 'gateway4': '1.2.3.5', + 'gateway6': IP6_GW, + 'nameservers': { + 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'], + 'search': ['example.com']}, + 'mtu': '1280'}, + 'enp0s25': { + 'match': {'macaddress': MAC_1}, + 'addresses': ['10.3.1.3/16'], + 'gateway4': '10.3.0.1', + 'gateway6': None, + 'nameservers': { + 'addresses': ['10.3.1.2', '1.2.3.8'], + 'search': [ + 'third.example.com', + 'third.example.org']}}}} self.assertEqual(expected, net.gen_conf()) -- cgit v1.2.3 From 7713713265224d9f34410028de6bd50d1451aed9 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Wed, 21 Mar 2018 20:07:29 -0600 Subject: Identify OpenTelekomCloud Xen as OpenStack DS. Open Telekom Cloud gen1 (Xen) hosts do not provide nova product names in DMI but Xen HVM domU. They can however be safely identified by the OpenTelekomCloud Chassis asset tag. OpenTelekomCloud does use the network OpenStack DataSource, so we better detect it. LP: #1756471 --- tests/unittests/test_ds_identify.py | 11 +++++++++++ tools/ds-identify | 4 ++++ 2 files changed, 15 insertions(+) (limited to 'tests') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 9c5628e7..85999b7a 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -66,6 +66,7 @@ P_DSID_CFG = "etc/cloud/ds-identify.cfg" MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} +MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} @@ -291,6 +292,10 @@ class TestDsIdentify(CiTestCase): """On Intel, openstack must be identified.""" self._test_ds_found('OpenStack') + def test_openstack_open_telekom_cloud(self): + """Open Telecom identification.""" + self._test_ds_found('OpenStack-OpenTelekom') + def test_openstack_on_non_intel_is_maybe(self): """On non-Intel, openstack without dmi info is maybe. @@ -502,6 +507,12 @@ VALID_CFG = { 'policy_dmi': POLICY_FOUND_ONLY, 'policy_no_dmi': POLICY_FOUND_ONLY, }, + 'OpenStack-OpenTelekom': { + # OTC gen1 (Xen) hosts use OpenStack datasource, LP: #1756471 + 'ds': 'OpenStack', + 'files': {P_CHASSIS_ASSET_TAG: 'OpenTelekomCloud\n'}, + 'mocks': [MOCK_VIRT_IS_XEN], + }, 'OVF-seed': { 'ds': 'OVF', 'files': { diff --git a/tools/ds-identify b/tools/ds-identify index e3f93c90..e2552c8b 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -894,6 +894,10 @@ dscheck_OpenStack() { return ${DS_FOUND} fi + if dmi_chassis_asset_tag_matches "OpenTelekomCloud"; then + return ${DS_FOUND} + fi + # LP: #1715241 : arch other than intel are not identified properly. case "$DI_UNAME_MACHINE" in i?86|x86_64) :;; -- cgit v1.2.3 From d29eeccd2c422b8eb3b053fc13ca966ed6d74c78 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 22 Mar 2018 09:55:27 -0600 Subject: Handle global dns entries in netplan In network config v1 format, there are dns values which are not bound to a specific interface and do not map to the per-interface format in netplan. To handle this case we render netplan configuration that duplicates the DNS configuration on any interface that has a static network config. We avoiding interfaces which have DHCP configuration which may provide conflicting DNS values. LP: #1750884 --- cloudinit/net/netplan.py | 25 ++++++--------- tests/unittests/test_net.py | 75 ++++++++------------------------------------- 2 files changed, 21 insertions(+), 79 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 6bee3d37..63443484 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -336,22 +336,15 @@ class Renderer(renderer.Renderer): _extract_addresses(ifcfg, vlan) vlans.update({ifname: vlan}) - # inject global nameserver values under each physical interface - if nameservers: - for _eth, cfg in ethernets.items(): - nscfg = cfg.get('nameservers', {}) - addresses = nscfg.get('addresses', []) - addresses += nameservers - nscfg.update({'addresses': addresses}) - cfg.update({'nameservers': nscfg}) - - if searchdomains: - for _eth, cfg in ethernets.items(): - nscfg = cfg.get('nameservers', {}) - search = nscfg.get('search', []) - search += searchdomains - nscfg.update({'search': search}) - cfg.update({'nameservers': nscfg}) + # inject global nameserver values under each all interface which + # has addresses and do not already have a DNS configuration + if nameservers or searchdomains: + nscfg = {'addresses': nameservers, 'search': searchdomains} + for section in [ethernets, wifis, bonds, bridges, vlans]: + for _name, cfg in section.items(): + if 'nameservers' in cfg or 'addresses' not in cfg: + continue + cfg.update({'nameservers': nscfg}) # workaround yaml dictionary key sorting when dumping def _render_section(name, section): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 84a0eabf..c12a487a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -393,12 +393,6 @@ NETWORK_CONFIGS = { eth1: match: macaddress: cf:d6:af:48:e8:80 - nameservers: - addresses: - - 1.2.3.4 - - 5.6.7.8 - search: - - wark.maas set-name: eth1 eth99: addresses: @@ -410,12 +404,9 @@ NETWORK_CONFIGS = { addresses: - 8.8.8.8 - 8.8.4.4 - - 1.2.3.4 - - 5.6.7.8 search: - barley.maas - sach.maas - - wark.maas routes: - to: 0.0.0.0/0 via: 65.61.151.37 @@ -654,81 +645,27 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true eth0: match: macaddress: c0:d6:9f:2c:e8:80 - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth0 eth1: match: macaddress: aa:d6:9f:2c:e8:80 - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth1 eth2: match: macaddress: c0:bb:9f:2c:e8:80 - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth2 eth3: match: macaddress: 66:bb:9f:2c:e8:80 - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth3 eth4: match: macaddress: 98:bb:9f:2c:e8:80 - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth4 eth5: dhcp4: true match: macaddress: 98:bb:9f:2c:e8:8a - nameservers: - addresses: - - 8.8.8.8 - - 4.4.4.4 - - 8.8.4.4 - search: - - barley.maas - - wark.maas - - foobar.maas set-name: eth5 bonds: bond0: @@ -748,6 +685,15 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true interfaces: - eth3 - eth4 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas parameters: ageing-time: 250 forward-delay: 1 @@ -2334,6 +2280,9 @@ class TestNetplanRoundTrip(CiTestCase): def testsimple_render_all(self): entry = NETWORK_CONFIGS['all'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + print(entry['expected_netplan']) + print('-- expected ^ | v rendered --') + print(files['/etc/netplan/50-cloud-init.yaml']) self.assertEqual( entry['expected_netplan'].splitlines(), files['/etc/netplan/50-cloud-init.yaml'].splitlines()) -- cgit v1.2.3 From 0d51e912146b3031c458ce415b7d4cd6eb17d06e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 22 Mar 2018 21:13:06 -0400 Subject: ubuntu-advantage: Add new config module to support ubuntu-advantage-tools ubuntu-advantage-tools is a package for enabling and disabling extended support services such as Extended Security Maintenance (ESM), Canonical Livepatch and FIPS certified PPAs. Simplify Ubuntu Advantage setup on machines by allowing users to provide a list of ubuntu-advantage commands in cloud-config. --- cloudinit/config/cc_snap.py | 47 +---- cloudinit/config/cc_ubuntu_advantage.py | 173 +++++++++++++++ cloudinit/config/tests/test_snap.py | 52 +---- cloudinit/config/tests/test_ubuntu_advantage.py | 268 ++++++++++++++++++++++++ cloudinit/subp.py | 57 +++++ cloudinit/tests/test_subp.py | 61 ++++++ config/cloud.cfg.tmpl | 3 + doc/rtd/topics/modules.rst | 1 + tests/unittests/test_handler/test_schema.py | 1 + 9 files changed, 568 insertions(+), 95 deletions(-) create mode 100644 cloudinit/config/cc_ubuntu_advantage.py create mode 100644 cloudinit/config/tests/test_ubuntu_advantage.py create mode 100644 cloudinit/subp.py create mode 100644 cloudinit/tests/test_subp.py (limited to 'tests') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index db965291..34a53fd4 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -11,6 +11,7 @@ from cloudinit import log as logging from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import prepend_base_command from cloudinit import util @@ -160,50 +161,6 @@ def add_assertions(assertions): util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) -def prepend_snap_commands(commands): - """Ensure user-provided commands start with SNAP_CMD, warn otherwise. - - Each command is either a list or string. Perform the following: - - When the command is a list, pop the first element if it is None - - When the command is a list, insert SNAP_CMD as the first element if - not present. - - When the command is a string containing a non-snap command, warn. - - Support cut-n-paste snap command sets from public snappy documentation. - Allow flexibility to provide non-snap environment/config setup if needed. - - @commands: List of commands. Each command element is a list or string. - - @return: List of 'fixed up' snap commands. - @raise: TypeError on invalid config item type. - """ - warnings = [] - errors = [] - fixed_commands = [] - for command in commands: - if isinstance(command, list): - if command[0] is None: # Avoid warnings by specifying None - command = command[1:] - elif command[0] != SNAP_CMD: # Automatically prepend SNAP_CMD - command.insert(0, SNAP_CMD) - elif isinstance(command, str): - if not command.startswith('%s ' % SNAP_CMD): - warnings.append(command) - else: - errors.append(str(command)) - continue - fixed_commands.append(command) - - if warnings: - LOG.warning( - 'Non-snap commands in snap config:\n%s', '\n'.join(warnings)) - if errors: - raise TypeError( - 'Invalid snap config.' - ' These commands are not a string or list:\n' + '\n'.join(errors)) - return fixed_commands - - def run_commands(commands): """Run the provided commands provided in snap:commands configuration. @@ -224,7 +181,7 @@ def run_commands(commands): 'commands parameter was not a list or dict: {commands}'.format( commands=commands)) - fixed_snap_commands = prepend_snap_commands(commands) + fixed_snap_commands = prepend_base_command('snap', commands) cmd_failures = [] for command in fixed_snap_commands: diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py new file mode 100644 index 00000000..16b1868b --- /dev/null +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -0,0 +1,173 @@ +# Copyright (C) 2018 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Ubuntu advantage: manage ubuntu-advantage offerings from Canonical.""" + +import sys +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import prepend_base_command +from cloudinit import util + + +distros = ['ubuntu'] +frequency = PER_INSTANCE + +LOG = logging.getLogger(__name__) + +schema = { + 'id': 'cc_ubuntu_advantage', + 'name': 'Ubuntu Advantage', + 'title': 'Install, configure and manage ubuntu-advantage offerings', + 'description': dedent("""\ + This module provides configuration options to setup ubuntu-advantage + subscriptions. + + .. note:: + Both ``commands`` value can be either a dictionary or a list. If + the configuration provided is a dictionary, the keys are only used + to order the execution of the commands and the dictionary is + merged with any vendor-data ubuntu-advantage configuration + provided. If a ``commands`` is provided as a list, any vendor-data + ubuntu-advantage ``commands`` are ignored. + + Ubuntu-advantage ``commands`` is a dictionary or list of + ubuntu-advantage commands to run on the deployed machine. + These commands can be used to enable or disable subscriptions to + various ubuntu-advantage products. See 'man ubuntu-advantage' for more + information on supported subcommands. + + .. note:: + Each command item can be a string or list. If the item is a list, + 'ubuntu-advantage' can be omitted and it will automatically be + inserted as part of the command. + """), + 'distros': distros, + 'examples': [dedent("""\ + # Enable Extended Security Maintenance using your service auth token + ubuntu-advantage: + commands: + 00: ubuntu-advantage enable-esm + """), dedent("""\ + # Enable livepatch by providing your livepatch token + ubuntu-advantage: + commands: + 00: ubuntu-advantage enable-livepatch + + """), dedent("""\ + # Convenience: the ubuntu-advantage command can be omitted when + # specifying commands as a list and 'ubuntu-advantage' will + # automatically be prepended. + # The following commands are equivalent + ubuntu-advantage: + commands: + 00: ['enable-livepatch', 'my-token'] + 01: ['ubuntu-advantage', 'enable-livepatch', 'my-token'] + 02: ubuntu-advantage enable-livepatch my-token + 03: 'ubuntu-advantage enable-livepatch my-token' + """)], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'ubuntu-advantage': { + 'type': 'object', + 'properties': { + 'commands': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject non-string & non-list + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + } + }, + 'additionalProperties': False, # Reject keys not in schema + 'required': ['commands'] + } + } +} + +# TODO schema for 'assertions' and 'commands' are too permissive at the moment. +# Once python-jsonschema supports schema draft 6 add support for arbitrary +# object keys with 'patternProperties' constraint to validate string values. + +__doc__ = get_schema_doc(schema) # Supplement python help() + +UA_CMD = "ubuntu-advantage" + + +def run_commands(commands): + """Run the commands provided in ubuntu-advantage:commands config. + + Commands are run individually. Any errors are collected and reported + after attempting all commands. + + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ + if not commands: + return + LOG.debug('Running user-provided ubuntu-advantage commands') + if isinstance(commands, dict): + # Sort commands based on dictionary key + commands = [v for _, v in sorted(commands.items())] + elif not isinstance(commands, list): + raise TypeError( + 'commands parameter was not a list or dict: {commands}'.format( + commands=commands)) + + fixed_ua_commands = prepend_base_command('ubuntu-advantage', commands) + + cmd_failures = [] + for command in fixed_ua_commands: + shell = isinstance(command, str) + try: + util.subp(command, shell=shell, status_cb=sys.stderr.write) + except util.ProcessExecutionError as e: + cmd_failures.append(str(e)) + if cmd_failures: + msg = ( + 'Failures running ubuntu-advantage commands:\n' + '{cmd_failures}'.format( + cmd_failures=cmd_failures)) + util.logexc(LOG, msg) + raise RuntimeError(msg) + + +def maybe_install_ua_tools(cloud): + """Install ubuntu-advantage-tools if not present.""" + if util.which('ubuntu-advantage'): + return + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(LOG, "Package update failed") + raise + try: + cloud.distro.install_packages(['ubuntu-advantage-tools']) + except Exception as e: + util.logexc(LOG, "Failed to install ubuntu-advantage-tools") + raise + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('ubuntu-advantage') + if cfgin is None: + LOG.debug(("Skipping module named %s," + " no 'ubuntu-advantage' key in configuration"), name) + return + + validate_cloudconfig_schema(cfg, schema) + maybe_install_ua_tools(cloud) + run_commands(cfgin.get('commands', [])) + +# vi: ts=4 expandtab diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index cb1205e9..988e7f7c 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -4,8 +4,8 @@ import re from six import StringIO from cloudinit.config.cc_snap import ( - ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands, - maybe_install_squashfuse, run_commands, schema) + ASSERTIONS_FILE, add_assertions, handle, maybe_install_squashfuse, + run_commands, schema) from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call @@ -158,54 +158,6 @@ class TestAddAssertions(CiTestCase): util.load_file(compare_file), util.load_file(assert_file)) -class TestPrepentSnapCommands(CiTestCase): - - with_logs = True - - def test_prepend_snap_commands_errors_on_neither_string_nor_list(self): - """Raise an error for each command which is not a string or list.""" - orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']] - with self.assertRaises(TypeError) as context_manager: - prepend_snap_commands(orig_commands) - self.assertEqual( - "Invalid snap config. These commands are not a string or list:\n" - "1\n{'not': 'gonna work'}", - str(context_manager.exception)) - - def test_prepend_snap_commands_warns_on_non_snap_string_commands(self): - """Warn on each non-snap for commands of type string.""" - orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x'] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual( - 'WARNING: Non-snap commands in snap config:\n' - 'ls\ntouch /blah\n', - self.logs.getvalue()) - self.assertEqual(orig_commands, fixed_commands) - - def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self): - """Prepend 'snap' for each non-snap command of type list.""" - orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'], - ['snap', 'install', 'x']] - expected = [['snap', 'ls'], ['snap', 'list'], - ['snap', 'snapa', '/blah'], - ['snap', 'install', 'x']] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual('', self.logs.getvalue()) - self.assertEqual(expected, fixed_commands) - - def test_prepend_snap_commands_removes_first_item_when_none(self): - """Remove the first element of a non-snap command when it is None.""" - orig_commands = [[None, 'ls'], ['snap', 'list'], - [None, 'touch', '/blah'], - ['snap', 'install', 'x']] - expected = [['ls'], ['snap', 'list'], - ['touch', '/blah'], - ['snap', 'install', 'x']] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual('', self.logs.getvalue()) - self.assertEqual(expected, fixed_commands) - - class TestRunCommands(CiTestCase): with_logs = True diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py new file mode 100644 index 00000000..0eeadd43 --- /dev/null +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -0,0 +1,268 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re +from six import StringIO + +from cloudinit.config.cc_ubuntu_advantage import ( + handle, maybe_install_ua_tools, run_commands, schema) +from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase, mock + + +# Module path used in mocks +MPATH = 'cloudinit.config.cc_ubuntu_advantage' + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestRunCommands(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestRunCommands, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.util.subp' % MPATH) + def test_run_commands_on_empty_list(self, m_subp): + """When provided with an empty list, run_commands does nothing.""" + run_commands([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_run_commands_on_non_list_or_dict(self): + """When provided an invalid type, run_commands raises an error.""" + with self.assertRaises(TypeError) as context_manager: + run_commands(commands="I'm Not Valid") + self.assertEqual( + "commands parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + def test_run_command_logs_commands_and_exit_codes_to_stderr(self): + """All exit codes are logged to stderr.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'bogus command' + cmd3 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2, cmd3] + + mock_path = '%s.sys.stderr' % MPATH + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with self.assertRaises(RuntimeError) as context_manager: + run_commands(commands=commands) + + self.assertIsNotNone( + re.search(r'bogus: (command )?not found', + str(context_manager.exception)), + msg='Expected bogus command not found') + expected_stderr_log = '\n'.join([ + 'Begin run command: {cmd}'.format(cmd=cmd1), + 'End run command: exit(0)', + 'Begin run command: {cmd}'.format(cmd=cmd2), + 'ERROR: End run command: exit(127)', + 'Begin run command: {cmd}'.format(cmd=cmd3), + 'End run command: exit(0)\n']) + self.assertEqual(expected_stderr_log, m_stderr.getvalue()) + + def test_run_command_as_lists(self): + """When commands are specified as a list, run them in order.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2] + with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO): + run_commands(commands=commands) + + self.assertIn( + 'DEBUG: Running user-provided ubuntu-advantage commands', + self.logs.getvalue()) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + self.assertIn( + 'WARNING: Non-ubuntu-advantage commands in ubuntu-advantage' + ' config:', + self.logs.getvalue()) + + def test_run_command_dict_sorted_as_command_script(self): + """When commands are a dict, sort them and run.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = {'02': cmd1, '01': cmd2} + with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO): + run_commands(commands=commands) + + expected_messages = [ + 'DEBUG: Running user-provided ubuntu-advantage commands'] + for message in expected_messages: + self.assertIn(message, self.logs.getvalue()) + self.assertEqual('MOM\nHI\n', util.load_file(outfile)) + + +class TestSchema(CiTestCase): + + with_logs = True + + def test_schema_warns_on_ubuntu_advantage_not_as_dict(self): + """If ubuntu-advantage configuration is not a dict, emit a warning.""" + validate_cloudconfig_schema({'ubuntu-advantage': 'wrong type'}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage: 'wrong type' is not" + " of type 'object'\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_schema_disallows_unknown_keys(self, _): + """Unknown keys in ubuntu-advantage configuration emit warnings.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': ['ls'], 'invalid-key': ''}}, + schema) + self.assertIn( + 'WARNING: Invalid config:\nubuntu-advantage: Additional properties' + " are not allowed ('invalid-key' was unexpected)", + self.logs.getvalue()) + + def test_warn_schema_requires_commands(self): + """Warn when ubuntu-advantage configuration lacks commands.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage: 'commands' is a" + " required property\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_warn_schema_commands_is_not_list_or_dict(self, _): + """Warn when ubuntu-advantage:commands config is not a list or dict.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage.commands: 'broken' is" + " not of type 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_warn_schema_when_commands_is_empty(self, _): + """Emit warnings when ubuntu-advantage:commands is empty.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': []}}, schema) + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage.commands: [] is too" + " short\nWARNING: Invalid config:\nubuntu-advantage.commands: {}" + " does not have enough properties\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_schema_when_commands_are_list_or_dict(self, _): + """No warnings when ubuntu-advantage:commands are a list or dict.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': ['valid']}}, schema) + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + +class TestHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestHandle, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.run_commands' % MPATH) + @mock.patch('%s.validate_cloudconfig_schema' % MPATH) + def test_handle_no_config(self, m_schema, m_run): + """When no ua-related configuration is provided, nothing happens.""" + cfg = {} + handle('ua-test', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertIn( + "DEBUG: Skipping module named ua-test, no 'ubuntu-advantage' key" + " in config", + self.logs.getvalue()) + m_schema.assert_not_called() + m_run.assert_not_called() + + @mock.patch('%s.maybe_install_ua_tools' % MPATH) + def test_handle_tries_to_install_ubuntu_advantage_tools(self, m_install): + """If ubuntu_advantage is provided, try installing ua-tools package.""" + cfg = {'ubuntu-advantage': {}} + mycloud = FakeCloud(None) + handle('nomatter', cfg=cfg, cloud=mycloud, log=self.logger, args=None) + m_install.assert_called_once_with(mycloud) + + @mock.patch('%s.maybe_install_ua_tools' % MPATH) + def test_handle_runs_commands_provided(self, m_install): + """When commands are specified as a list, run them.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cfg = { + 'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile, + 'echo "MOM" >> %s' % outfile]}} + mock_path = '%s.sys.stderr' % MPATH + with mock.patch(mock_path, new_callable=StringIO): + handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + + +class TestMaybeInstallUATools(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestMaybeInstallUATools, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): + """Do nothing if ubuntu-advantage-tools already exists.""" + m_which.return_value = '/usr/bin/ubuntu-advantage' # already installed + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_raises_update_errors(self, m_which): + """maybe_install_ua_tools logs and raises apt update errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_ua_tools(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_raises_install_errors(self, m_which): + """maybe_install_ua_tools logs and raises package install errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.return_value = None + distro.install_packages.side_effect = RuntimeError( + 'Some install error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_ua_tools(cloud=FakeCloud(distro)) + self.assertEqual('Some install error', str(context_manager.exception)) + self.assertIn( + 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_happy_path(self, m_which): + """maybe_install_ua_tools installs ubuntu-advantage-tools.""" + m_which.return_value = None + distro = mock.MagicMock() # No errors raised + maybe_install_ua_tools(cloud=FakeCloud(distro)) + distro.update_package_sources.assert_called_once_with() + distro.install_packages.assert_called_once_with( + ['ubuntu-advantage-tools']) + +# vi: ts=4 expandtab diff --git a/cloudinit/subp.py b/cloudinit/subp.py new file mode 100644 index 00000000..0ad09306 --- /dev/null +++ b/cloudinit/subp.py @@ -0,0 +1,57 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Common utility functions for interacting with subprocess.""" + +# TODO move subp shellify and runparts related functions out of util.py + +import logging + +LOG = logging.getLogger(__name__) + + +def prepend_base_command(base_command, commands): + """Ensure user-provided commands start with base_command; warn otherwise. + + Each command is either a list or string. Perform the following: + - If the command is a list, pop the first element if it is None + - If the command is a list, insert base_command as the first element if + not present. + - When the command is a string not starting with 'base-command', warn. + + Allow flexibility to provide non-base-command environment/config setup if + needed. + + @commands: List of commands. Each command element is a list or string. + + @return: List of 'fixed up' commands. + @raise: TypeError on invalid config item type. + """ + warnings = [] + errors = [] + fixed_commands = [] + for command in commands: + if isinstance(command, list): + if command[0] is None: # Avoid warnings by specifying None + command = command[1:] + elif command[0] != base_command: # Automatically prepend + command.insert(0, base_command) + elif isinstance(command, str): + if not command.startswith('%s ' % base_command): + warnings.append(command) + else: + errors.append(str(command)) + continue + fixed_commands.append(command) + + if warnings: + LOG.warning( + 'Non-%s commands in %s config:\n%s', + base_command, base_command, '\n'.join(warnings)) + if errors: + raise TypeError( + 'Invalid {name} config.' + ' These commands are not a string or list:\n{errors}'.format( + name=base_command, errors='\n'.join(errors))) + return fixed_commands + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py new file mode 100644 index 00000000..448097d3 --- /dev/null +++ b/cloudinit/tests/test_subp.py @@ -0,0 +1,61 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Tests for cloudinit.subp utility functions""" + +from cloudinit import subp +from cloudinit.tests.helpers import CiTestCase + + +class TestPrependBaseCommands(CiTestCase): + + with_logs = True + + def test_prepend_base_command_errors_on_neither_string_nor_list(self): + """Raise an error for each command which is not a string or list.""" + orig_commands = ['ls', 1, {'not': 'gonna work'}, ['basecmd', 'list']] + with self.assertRaises(TypeError) as context_manager: + subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual( + "Invalid basecmd config. These commands are not a string or" + " list:\n1\n{'not': 'gonna work'}", + str(context_manager.exception)) + + def test_prepend_base_command_warns_on_non_base_string_commands(self): + """Warn on each non-base for commands of type string.""" + orig_commands = [ + 'ls', 'basecmd list', 'touch /blah', 'basecmd install x'] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual( + 'WARNING: Non-basecmd commands in basecmd config:\n' + 'ls\ntouch /blah\n', + self.logs.getvalue()) + self.assertEqual(orig_commands, fixed_commands) + + def test_prepend_base_command_prepends_on_non_base_list_commands(self): + """Prepend 'basecmd' for each non-basecmd command of type list.""" + orig_commands = [['ls'], ['basecmd', 'list'], ['basecmda', '/blah'], + ['basecmd', 'install', 'x']] + expected = [['basecmd', 'ls'], ['basecmd', 'list'], + ['basecmd', 'basecmda', '/blah'], + ['basecmd', 'install', 'x']] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + def test_prepend_base_command_removes_first_item_when_none(self): + """Remove the first element of a non-basecmd when it is None.""" + orig_commands = [[None, 'ls'], ['basecmd', 'list'], + [None, 'touch', '/blah'], + ['basecmd', 'install', 'x']] + expected = [['ls'], ['basecmd', 'list'], + ['touch', '/blah'], + ['basecmd', 'install', 'x']] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + +# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 56a34fab..3129d4eb 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -87,6 +87,9 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} +{% if variant in ["ubuntu"] %} + - ubuntu-advantage +{% endif %} {% if variant in ["suse"] %} - zypper-add-repo {% endif %} diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index a0f68129..d9720f6a 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -53,6 +53,7 @@ Modules .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone +.. automodule:: cloudinit.config.cc_ubuntu_advantage .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 9b50ee79..ac41f124 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,6 +27,7 @@ class GetSchemaTest(CiTestCase): 'cc_resizefs', 'cc_runcmd', 'cc_snap', + 'cc_ubuntu_advantage', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) -- cgit v1.2.3 From 2d618e27687470a8a3649f44598819bdee8cdb03 Mon Sep 17 00:00:00 2001 From: Douglas Jordan Date: Fri, 23 Mar 2018 12:04:18 -0600 Subject: Reduce AzurePreprovisioning HTTP timeouts. Reducing timeout to 1 second as IMDS responds within a handful of milliseconds. Also get rid of max_retries to prevent exiting out of polling loop early due to IMDS outage / upgrade. Reduce Azure PreProvisioning HTTP timeouts during polling to avoid waiting an extra minute. LP: #1752977 --- cloudinit/sources/DataSourceAzure.py | 31 ++++++++------------------- cloudinit/url_helper.py | 13 ++++++----- tests/unittests/test_datasource/test_azure.py | 22 ++++++------------- 3 files changed, 24 insertions(+), 42 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 0bb7fad9..0ee622e2 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -20,7 +20,7 @@ from cloudinit import net from cloudinit.net.dhcp import EphemeralDHCPv4 from cloudinit import sources from cloudinit.sources.helpers.azure import get_metadata_from_fabric -from cloudinit.url_helper import readurl, wait_for_url, UrlError +from cloudinit.url_helper import readurl, UrlError from cloudinit import util LOG = logging.getLogger(__name__) @@ -49,7 +49,6 @@ DEFAULT_FS = 'ext4' AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77' REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds" IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata" -IMDS_RETRIES = 5 def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid): @@ -451,36 +450,24 @@ class DataSourceAzure(sources.DataSource): headers = {"Metadata": "true"} LOG.debug("Start polling IMDS") - def sleep_cb(response, loop_n): - return 1 - - def exception_cb(msg, exception): + def exc_cb(msg, exception): if isinstance(exception, UrlError) and exception.code == 404: - return - LOG.warning("Exception during polling. Will try DHCP.", - exc_info=True) - + return True # If we get an exception while trying to call IMDS, we # call DHCP and setup the ephemeral network to acquire the new IP. - raise exception + return False need_report = report_ready - for i in range(IMDS_RETRIES): + while True: try: with EphemeralDHCPv4() as lease: if need_report: self._report_ready(lease=lease) need_report = False - wait_for_url([url], max_wait=None, timeout=60, - status_cb=LOG.info, - headers_cb=lambda url: headers, sleep_time=1, - exception_cb=exception_cb, - sleep_time_cb=sleep_cb) - return str(readurl(url, headers=headers)) - except Exception: - LOG.debug("Exception during polling-retrying dhcp" + - " %d more time(s).", (IMDS_RETRIES - i), - exc_info=True) + return readurl(url, timeout=1, headers=headers, + exception_cb=exc_cb, infinite=True).contents + except UrlError: + pass def _report_ready(self, lease): """Tells the fabric provisioning has completed diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 36289af5..03a573af 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -16,7 +16,7 @@ import time from email.utils import parsedate from functools import partial - +from itertools import count from requests import exceptions from six.moves.urllib.parse import ( @@ -172,7 +172,7 @@ def _get_ssl_args(url, ssl_details): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, check_status=True, allow_redirects=True, exception_cb=None, - session=None): + session=None, infinite=False): url = _cleanurl(url) req_args = { 'url': url, @@ -220,7 +220,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, excps = [] # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... - for i in range(0, manual_tries): + # Infinitely retry if infinite is True + for i in count() if infinite else range(0, manual_tries): req_args['headers'] = headers_cb(url) filtered_req_args = {} for (k, v) in req_args.items(): @@ -229,7 +230,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, filtered_req_args[k] = v try: LOG.debug("[%s/%s] open '%s' with %s configuration", i, - manual_tries, url, filtered_req_args) + "infinite" if infinite else manual_tries, url, + filtered_req_args) if session is None: session = requests.Session() @@ -263,7 +265,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # to continue retrying and False to break and re-raise the # exception break - if i + 1 < manual_tries and sec_between > 0: + if (infinite and sec_between > 0) or \ + (i + 1 < manual_tries and sec_between > 0): LOG.debug("Please wait %s seconds while we wait to try again", sec_between) time.sleep(sec_between) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index da7da0ca..3e8b7913 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1177,7 +1177,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02' host = "169.254.169.254" full_url = url.format(host) - fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf") + fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf", + content="ovf") dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) self.assertTrue(len(dsa._poll_imds()) > 0) self.assertEqual(fake_resp.call_args_list, @@ -1185,13 +1186,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): headers={'Metadata': 'true', 'User-Agent': 'Cloud-Init/%s' % vs() - }, method='GET', timeout=60.0, - url=full_url), - mock.call(allow_redirects=True, - headers={'Metadata': 'true', - 'User-Agent': - 'Cloud-Init/%s' % vs() - }, method='GET', url=full_url)]) + }, method='GET', timeout=1, + url=full_url)]) self.assertEqual(m_dhcp.call_count, 1) m_net.assert_any_call( broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', @@ -1217,7 +1213,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): username = "myuser" odata = {'HostName': hostname, 'UserName': username} content = construct_valid_ovf_env(data=odata) - fake_resp.return_value = mock.MagicMock(status_code=200, text=content) + fake_resp.return_value = mock.MagicMock(status_code=200, text=content, + content=content) dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) md, ud, cfg, d = dsa._reprovision() self.assertEqual(md['local-hostname'], hostname) @@ -1227,12 +1224,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): headers={'Metadata': 'true', 'User-Agent': 'Cloud-Init/%s' % vs()}, - method='GET', timeout=60.0, url=full_url), - mock.call(allow_redirects=True, - headers={'Metadata': 'true', - 'User-Agent': - 'Cloud-Init/%s' % vs()}, - method='GET', url=full_url)]) + method='GET', timeout=1, url=full_url)]) self.assertEqual(m_dhcp.call_count, 1) m_net.assert_any_call( broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', -- cgit v1.2.3 From e0f644b7c8c76bd63d242558685722cc70d9c51d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 23 Mar 2018 17:17:55 -0600 Subject: IBMCloud: Initial IBM Cloud datasource. This adds a specific IBM Cloud datasource. IBM Cloud is identified by: a.) running on xen b.) one of a LABEL=METADATA disk or a LABEL=config-2 disk with UUID=9796-932E The datasource contains its own config-drive reader that reads only the currently supported portion of config-drive needed for ibm cloud. During the provisioning boot, cloud-init is disabled. See the docstring in DataSourceIBMCloud.py for more more information. --- cloudinit/sources/DataSourceConfigDrive.py | 10 + cloudinit/sources/DataSourceIBMCloud.py | 325 +++++++++++++++++++++++ cloudinit/tests/test_util.py | 72 +++++ cloudinit/util.py | 31 +++ tests/unittests/test_datasource/test_ibmcloud.py | 262 ++++++++++++++++++ tests/unittests/test_ds_identify.py | 104 +++++++- tools/ds-identify | 65 ++++- 7 files changed, 857 insertions(+), 12 deletions(-) create mode 100644 cloudinit/sources/DataSourceIBMCloud.py create mode 100644 tests/unittests/test_datasource/test_ibmcloud.py (limited to 'tests') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index b8db6267..c7b5fe5f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -14,6 +14,7 @@ from cloudinit import util from cloudinit.net import eni +from cloudinit.sources.DataSourceIBMCloud import get_ibm_platform from cloudinit.sources.helpers import openstack LOG = logging.getLogger(__name__) @@ -255,6 +256,15 @@ def find_candidate_devs(probe_optical=True): # an unpartitioned block device (ex sda, not sda1) devices = [d for d in candidates if d in by_label or not util.is_partition(d)] + + if devices: + # IBMCloud uses config-2 label, but limited to a single UUID. + ibm_platform, ibm_path = get_ibm_platform() + if ibm_path in devices: + devices.remove(ibm_path) + LOG.debug("IBMCloud device '%s' (%s) removed from candidate list", + ibm_path, ibm_platform) + return devices diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py new file mode 100644 index 00000000..02b3d56f --- /dev/null +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -0,0 +1,325 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Datasource for IBMCloud. + +IBMCloud is also know as SoftLayer or BlueMix. +IBMCloud hypervisor is xen (2018-03-10). + +There are 2 different api exposed launch methods. + * template: This is the legacy method of launching instances. + When booting from an image template, the system boots first into + a "provisioning" mode. There, host <-> guest mechanisms are utilized + to execute code in the guest and provision it. + + Cloud-init will disable itself when it detects that it is in the + provisioning mode. It detects this by the presence of + a file '/root/provisioningConfiguration.cfg'. + + When provided with user-data, the "first boot" will contain a + ConfigDrive-like disk labeled with 'METADATA'. If there is no user-data + provided, then there is no data-source. + + Cloud-init never does any network configuration in this mode. + + * os_code: Essentially "launch by OS Code" (Operating System Code). + This is a more modern approach. There is no specific "provisioning" boot. + Instead, cloud-init does all the customization. With or without + user-data provided, an OpenStack ConfigDrive like disk is attached. + + Only disks with label 'config-2' and UUID '9796-932E' are considered. + This is to avoid this datasource claiming ConfigDrive. This does + mean that 1 in 8^16 (~4 billion) Xen ConfigDrive systems will be + incorrectly identified as IBMCloud. + +TODO: + * is uuid (/sys/hypervisor/uuid) stable for life of an instance? + it seems it is not the same as data's uuid in the os_code case + but is in the template case. + +""" +import base64 +import json +import os + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit.sources.helpers import openstack +from cloudinit import util + +LOG = logging.getLogger(__name__) + +IBM_CONFIG_UUID = "9796-932E" + + +class Platforms(object): + TEMPLATE_LIVE_METADATA = "Template/Live/Metadata" + TEMPLATE_LIVE_NODATA = "UNABLE TO BE IDENTIFIED." + TEMPLATE_PROVISIONING_METADATA = "Template/Provisioning/Metadata" + TEMPLATE_PROVISIONING_NODATA = "Template/Provisioning/No-Metadata" + OS_CODE = "OS-Code/Live" + + +PROVISIONING = ( + Platforms.TEMPLATE_PROVISIONING_METADATA, + Platforms.TEMPLATE_PROVISIONING_NODATA) + + +class DataSourceIBMCloud(sources.DataSource): + + dsname = 'IBMCloud' + system_uuid = None + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceIBMCloud, self).__init__(sys_cfg, distro, paths) + self.source = None + self._network_config = None + self.network_json = None + self.platform = None + + def __str__(self): + root = super(DataSourceIBMCloud, self).__str__() + mstr = "%s [%s %s]" % (root, self.platform, self.source) + return mstr + + def _get_data(self): + results = read_md() + if results is None: + return False + + self.source = results['source'] + self.platform = results['platform'] + self.metadata = results['metadata'] + self.userdata_raw = results.get('userdata') + self.network_json = results.get('networkdata') + vd = results.get('vendordata') + self.vendordata_pure = vd + self.system_uuid = results['system-uuid'] + try: + self.vendordata_raw = sources.convert_vendordata(vd) + except ValueError as e: + LOG.warning("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None + + return True + + def check_instance_id(self, sys_cfg): + """quickly (local check only) if self.instance_id is still valid + + in Template mode, the system uuid (/sys/hypervisor/uuid) is the + same as found in the METADATA disk. But that is not true in OS_CODE + mode. So we read the system_uuid and keep that for later compare.""" + if self.system_uuid is None: + return False + return self.system_uuid == _read_system_uuid() + + @property + def network_config(self): + if self.platform != Platforms.OS_CODE: + # If deployed from template, an agent in the provisioning + # environment handles networking configuration. Not cloud-init. + return {'config': 'disabled', 'version': 1} + if self._network_config is None: + if self.network_json is not None: + LOG.debug("network config provided via network_json") + self._network_config = openstack.convert_net_json( + self.network_json, known_macs=None) + else: + LOG.debug("no network configuration available.") + return self._network_config + + +def _read_system_uuid(): + uuid_path = "/sys/hypervisor/uuid" + if not os.path.isfile(uuid_path): + return None + return util.load_file(uuid_path).strip().lower() + + +def _is_xen(): + return os.path.exists("/proc/xen") + + +def _is_ibm_provisioning(): + return os.path.exists("/root/provisioningConfiguration.cfg") + + +def get_ibm_platform(): + """Return a tuple (Platform, path) + + If this is Not IBM cloud, then the return value is (None, None). + An instance in provisioning mode is considered running on IBM cloud.""" + label_mdata = "METADATA" + label_cfg2 = "CONFIG-2" + not_found = (None, None) + + if not _is_xen(): + return not_found + + # fslabels contains only the first entry with a given label. + fslabels = {} + try: + devs = util.blkid() + except util.ProcessExecutionError as e: + LOG.warning("Failed to run blkid: %s", e) + return (None, None) + + for dev in sorted(devs.keys()): + data = devs[dev] + label = data.get("LABEL", "").upper() + uuid = data.get("UUID", "").upper() + if label not in (label_mdata, label_cfg2): + continue + if label in fslabels: + LOG.warning("Duplicate fslabel '%s'. existing=%s current=%s", + label, fslabels[label], data) + continue + if label == label_cfg2 and uuid != IBM_CONFIG_UUID: + LOG.debug("Skipping %s with LABEL=%s due to uuid != %s: %s", + dev, label, uuid, data) + continue + fslabels[label] = data + + metadata_path = fslabels.get(label_mdata, {}).get('DEVNAME') + cfg2_path = fslabels.get(label_cfg2, {}).get('DEVNAME') + + if cfg2_path: + return (Platforms.OS_CODE, cfg2_path) + elif metadata_path: + if _is_ibm_provisioning(): + return (Platforms.TEMPLATE_PROVISIONING_METADATA, metadata_path) + else: + return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path) + elif _is_ibm_provisioning(): + return (Platforms.TEMPLATE_PROVISIONING_NODATA, None) + return not_found + + +def read_md(): + """Read data from IBM Cloud. + + @return: None if not running on IBM Cloud. + dictionary with guaranteed fields: metadata, version + and optional fields: userdata, vendordata, networkdata. + Also includes the system uuid from /sys/hypervisor/uuid.""" + platform, path = get_ibm_platform() + if platform is None: + LOG.debug("This is not an IBMCloud platform.") + return None + elif platform in PROVISIONING: + LOG.debug("Cloud-init is disabled during provisioning: %s.", + platform) + return None + + ret = {'platform': platform, 'source': path, + 'system-uuid': _read_system_uuid()} + + try: + if os.path.isdir(path): + results = metadata_from_dir(path) + else: + results = util.mount_cb(path, metadata_from_dir) + except BrokenMetadata as e: + raise RuntimeError( + "Failed reading IBM config disk (platform=%s path=%s): %s" % + (platform, path, e)) + + ret.update(results) + return ret + + +class BrokenMetadata(IOError): + pass + + +def metadata_from_dir(source_dir): + """Walk source_dir extracting standardized metadata. + + Certain metadata keys are renamed to present a standardized set of metadata + keys. + + This function has a lot in common with ConfigDriveReader.read_v2 but + there are a number of inconsistencies, such key renames and as only + presenting a 'latest' version which make it an unlikely candidate to share + code. + + @return: Dict containing translated metadata, userdata, vendordata, + networkdata as present. + """ + + def opath(fname): + return os.path.join("openstack", "latest", fname) + + def load_json_bytes(blob): + return json.loads(blob.decode('utf-8')) + + files = [ + # tuples of (results_name, path, translator) + ('metadata_raw', opath('meta_data.json'), load_json_bytes), + ('userdata', opath('user_data'), None), + ('vendordata', opath('vendor_data.json'), load_json_bytes), + ('networkdata', opath('network_data.json'), load_json_bytes), + ] + + results = {} + for (name, path, transl) in files: + fpath = os.path.join(source_dir, path) + raw = None + try: + raw = util.load_file(fpath, decode=False) + except IOError as e: + LOG.debug("Failed reading path '%s': %s", fpath, e) + + if raw is None or transl is None: + data = raw + else: + try: + data = transl(raw) + except Exception as e: + raise BrokenMetadata("Failed decoding %s: %s" % (path, e)) + + results[name] = data + + if results.get('metadata_raw') is None: + raise BrokenMetadata( + "%s missing required file 'meta_data.json'" % source_dir) + + results['metadata'] = {} + + md_raw = results['metadata_raw'] + md = results['metadata'] + if 'random_seed' in md_raw: + try: + md['random_seed'] = base64.b64decode(md_raw['random_seed']) + except (ValueError, TypeError) as e: + raise BrokenMetadata( + "Badly formatted metadata random_seed entry: %s" % e) + + renames = ( + ('public_keys', 'public-keys'), ('hostname', 'local-hostname'), + ('uuid', 'instance-id')) + for mdname, newname in renames: + if mdname in md_raw: + md[newname] = md_raw[mdname] + + return results + + +# Used to match classes to dependencies +datasources = [ + (DataSourceIBMCloud, (sources.DEP_FILESYSTEM,)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description='Query IBM Cloud Metadata') + args = parser.parse_args() + data = read_md() + print(util.json_dumps(data)) + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index d30643dc..3f37dbb6 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -3,6 +3,7 @@ """Tests for cloudinit.util""" import logging +from textwrap import dedent import cloudinit.util as util @@ -140,4 +141,75 @@ class TestGetHostnameFqdn(CiTestCase): [{'fqdn': True, 'metadata_only': True}, {'metadata_only': True}], mycloud.calls) + +class TestBlkid(CiTestCase): + ids = { + "id01": "1111-1111", + "id02": "22222222-2222", + "id03": "33333333-3333", + "id04": "44444444-4444", + "id05": "55555555-5555-5555-5555-555555555555", + "id06": "66666666-6666-6666-6666-666666666666", + "id07": "52894610484658920398", + "id08": "86753098675309867530", + "id09": "99999999-9999-9999-9999-999999999999", + } + + blkid_out = dedent("""\ + /dev/loop0: TYPE="squashfs" + /dev/loop1: TYPE="squashfs" + /dev/loop2: TYPE="squashfs" + /dev/loop3: TYPE="squashfs" + /dev/sda1: UUID="{id01}" TYPE="vfat" PARTUUID="{id02}" + /dev/sda2: UUID="{id03}" TYPE="ext4" PARTUUID="{id04}" + /dev/sda3: UUID="{id05}" TYPE="ext4" PARTUUID="{id06}" + /dev/sda4: LABEL="default" UUID="{id07}" UUID_SUB="{id08}" """ + """TYPE="zfs_member" PARTUUID="{id09}" + /dev/loop4: TYPE="squashfs" + """) + + maxDiff = None + + def _get_expected(self): + return ({ + "/dev/loop0": {"DEVNAME": "/dev/loop0", "TYPE": "squashfs"}, + "/dev/loop1": {"DEVNAME": "/dev/loop1", "TYPE": "squashfs"}, + "/dev/loop2": {"DEVNAME": "/dev/loop2", "TYPE": "squashfs"}, + "/dev/loop3": {"DEVNAME": "/dev/loop3", "TYPE": "squashfs"}, + "/dev/loop4": {"DEVNAME": "/dev/loop4", "TYPE": "squashfs"}, + "/dev/sda1": {"DEVNAME": "/dev/sda1", "TYPE": "vfat", + "UUID": self.ids["id01"], + "PARTUUID": self.ids["id02"]}, + "/dev/sda2": {"DEVNAME": "/dev/sda2", "TYPE": "ext4", + "UUID": self.ids["id03"], + "PARTUUID": self.ids["id04"]}, + "/dev/sda3": {"DEVNAME": "/dev/sda3", "TYPE": "ext4", + "UUID": self.ids["id05"], + "PARTUUID": self.ids["id06"]}, + "/dev/sda4": {"DEVNAME": "/dev/sda4", "TYPE": "zfs_member", + "LABEL": "default", + "UUID": self.ids["id07"], + "UUID_SUB": self.ids["id08"], + "PARTUUID": self.ids["id09"]}, + }) + + @mock.patch("cloudinit.util.subp") + def test_functional_blkid(self, m_subp): + m_subp.return_value = ( + self.blkid_out.format(**self.ids), "") + self.assertEqual(self._get_expected(), util.blkid()) + m_subp.assert_called_with(["blkid", "-o", "full"], capture=True, + decode="replace") + + @mock.patch("cloudinit.util.subp") + def test_blkid_no_cache_uses_no_cache(self, m_subp): + """blkid should turn off cache if disable_cache is true.""" + m_subp.return_value = ( + self.blkid_out.format(**self.ids), "") + self.assertEqual(self._get_expected(), + util.blkid(disable_cache=True)) + m_subp.assert_called_with(["blkid", "-o", "full", "-c", "/dev/null"], + capture=True, decode="replace") + + # vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index cae8b196..fb4ee5fe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1237,6 +1237,37 @@ def find_devs_with(criteria=None, oformat='device', return entries +def blkid(devs=None, disable_cache=False): + """Get all device tags details from blkid. + + @param devs: Optional list of device paths you wish to query. + @param disable_cache: Bool, set True to start with clean cache. + + @return: Dict of key value pairs of info for the device. + """ + if devs is None: + devs = [] + else: + devs = list(devs) + + cmd = ['blkid', '-o', 'full'] + if disable_cache: + cmd.extend(['-c', '/dev/null']) + cmd.extend(devs) + + # we have to decode with 'replace' as shelx.split (called by + # load_shell_content) can't take bytes. So this is potentially + # lossy of non-utf-8 chars in blkid output. + out, _ = subp(cmd, capture=True, decode="replace") + ret = {} + for line in out.splitlines(): + dev, _, data = line.partition(":") + ret[dev] = load_shell_content(data) + ret[dev]["DEVNAME"] = dev + + return ret + + def peek_file(fname, max_bytes): LOG.debug("Peeking at %s (max_bytes=%s)", fname, max_bytes) with open(fname, 'rb') as ifh: diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py new file mode 100644 index 00000000..621cfe49 --- /dev/null +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -0,0 +1,262 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.sources import DataSourceIBMCloud as ibm +from cloudinit.tests import helpers as test_helpers + +import base64 +import copy +import json +import mock +from textwrap import dedent + +D_PATH = "cloudinit.sources.DataSourceIBMCloud." + + +class TestIBMCloud(test_helpers.CiTestCase): + """Test the datasource.""" + def setUp(self): + super(TestIBMCloud, self).setUp() + pass + + +@mock.patch(D_PATH + "_is_xen", return_value=True) +@mock.patch(D_PATH + "_is_ibm_provisioning") +@mock.patch(D_PATH + "util.blkid") +class TestGetIBMPlatform(test_helpers.CiTestCase): + """Test the get_ibm_platform helper.""" + + blkid_base = { + "/dev/xvda1": { + "DEVNAME": "/dev/xvda1", "LABEL": "cloudimg-bootfs", + "TYPE": "ext3"}, + "/dev/xvda2": { + "DEVNAME": "/dev/xvda2", "LABEL": "cloudimg-rootfs", + "TYPE": "ext4"}, + } + + blkid_metadata_disk = { + "/dev/xvdh1": { + "DEVNAME": "/dev/xvdh1", "LABEL": "METADATA", "TYPE": "vfat", + "SEC_TYPE": "msdos", "UUID": "681B-8C5D", + "PARTUUID": "3d631e09-01"}, + } + + blkid_oscode_disk = { + "/dev/xvdh": { + "DEVNAME": "/dev/xvdh", "LABEL": "config-2", "TYPE": "vfat", + "SEC_TYPE": "msdos", "UUID": ibm.IBM_CONFIG_UUID} + } + + def setUp(self): + self.blkid_metadata = copy.deepcopy(self.blkid_base) + self.blkid_metadata.update(copy.deepcopy(self.blkid_metadata_disk)) + + self.blkid_oscode = copy.deepcopy(self.blkid_base) + self.blkid_oscode.update(copy.deepcopy(self.blkid_oscode_disk)) + + def test_id_template_live_metadata(self, m_blkid, m_is_prov, _m_xen): + """identify TEMPLATE_LIVE_METADATA.""" + m_blkid.return_value = self.blkid_metadata + m_is_prov.return_value = False + self.assertEqual( + (ibm.Platforms.TEMPLATE_LIVE_METADATA, "/dev/xvdh1"), + ibm.get_ibm_platform()) + + def test_id_template_prov_metadata(self, m_blkid, m_is_prov, _m_xen): + """identify TEMPLATE_PROVISIONING_METADATA.""" + m_blkid.return_value = self.blkid_metadata + m_is_prov.return_value = True + self.assertEqual( + (ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh1"), + ibm.get_ibm_platform()) + + def test_id_template_prov_nodata(self, m_blkid, m_is_prov, _m_xen): + """identify TEMPLATE_PROVISIONING_NODATA.""" + m_blkid.return_value = self.blkid_base + m_is_prov.return_value = True + self.assertEqual( + (ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None), + ibm.get_ibm_platform()) + + def test_id_os_code(self, m_blkid, m_is_prov, _m_xen): + """Identify OS_CODE.""" + m_blkid.return_value = self.blkid_oscode + m_is_prov.return_value = False + self.assertEqual((ibm.Platforms.OS_CODE, "/dev/xvdh"), + ibm.get_ibm_platform()) + + def test_id_os_code_must_match_uuid(self, m_blkid, m_is_prov, _m_xen): + """Test against false positive on openstack with non-ibm UUID.""" + blkid = self.blkid_oscode + blkid["/dev/xvdh"]["UUID"] = "9999-9999" + m_blkid.return_value = blkid + m_is_prov.return_value = False + self.assertEqual((None, None), ibm.get_ibm_platform()) + + +@mock.patch(D_PATH + "_read_system_uuid", return_value=None) +@mock.patch(D_PATH + "get_ibm_platform") +class TestReadMD(test_helpers.CiTestCase): + """Test the read_datasource helper.""" + + template_md = { + "files": [], + "network_config": {"content_path": "/content/interfaces"}, + "hostname": "ci-fond-ram", + "name": "ci-fond-ram", + "domain": "testing.ci.cloud-init.org", + "meta": {"dsmode": "net"}, + "uuid": "8e636730-9f5d-c4a5-327c-d7123c46e82f", + "public_keys": {"1091307": "ssh-rsa AAAAB3NzaC1...Hw== ci-pubkey"}, + } + + oscode_md = { + "hostname": "ci-grand-gannet.testing.ci.cloud-init.org", + "name": "ci-grand-gannet", + "uuid": "2f266908-8e6c-4818-9b5c-42e9cc66a785", + "random_seed": "bm90LXJhbmRvbQo=", + "crypt_key": "ssh-rsa AAAAB3NzaC1yc2..n6z/", + "configuration_token": "eyJhbGciOi..M3ZA", + "public_keys": {"1091307": "ssh-rsa AAAAB3N..Hw== ci-pubkey"}, + } + + content_interfaces = dedent("""\ + auto lo + iface lo inet loopback + + auto eth0 + allow-hotplug eth0 + iface eth0 inet static + address 10.82.43.5 + netmask 255.255.255.192 + """) + + userdata = b"#!/bin/sh\necho hi mom\n" + # meta.js file gets json encoded userdata as a list. + meta_js = '["#!/bin/sh\necho hi mom\n"]' + vendor_data = { + "cloud-init": "#!/bin/bash\necho 'root:$6$5ab01p1m1' | chpasswd -e"} + + network_data = { + "links": [ + {"id": "interface_29402281", "name": "eth0", "mtu": None, + "type": "phy", "ethernet_mac_address": "06:00:f1:bd:da:25"}, + {"id": "interface_29402279", "name": "eth1", "mtu": None, + "type": "phy", "ethernet_mac_address": "06:98:5e:d0:7f:86"} + ], + "networks": [ + {"id": "network_109887563", "link": "interface_29402281", + "type": "ipv4", "ip_address": "10.82.43.2", + "netmask": "255.255.255.192", + "routes": [ + {"network": "10.0.0.0", "netmask": "255.0.0.0", + "gateway": "10.82.43.1"}, + {"network": "161.26.0.0", "netmask": "255.255.0.0", + "gateway": "10.82.43.1"}]}, + {"id": "network_109887551", "link": "interface_29402279", + "type": "ipv4", "ip_address": "108.168.194.252", + "netmask": "255.255.255.248", + "routes": [ + {"network": "0.0.0.0", "netmask": "0.0.0.0", + "gateway": "108.168.194.249"}]} + ], + "services": [ + {"type": "dns", "address": "10.0.80.11"}, + {"type": "dns", "address": "10.0.80.12"} + ], + } + + sysuuid = '7f79ebf5-d791-43c3-a723-854e8389d59f' + + def _get_expected_metadata(self, os_md): + """return expected 'metadata' for data loaded from meta_data.json.""" + os_md = copy.deepcopy(os_md) + renames = ( + ('hostname', 'local-hostname'), + ('uuid', 'instance-id'), + ('public_keys', 'public-keys')) + ret = {} + for osname, mdname in renames: + if osname in os_md: + ret[mdname] = os_md[osname] + if 'random_seed' in os_md: + ret['random_seed'] = base64.b64decode(os_md['random_seed']) + + return ret + + def test_provisioning_md(self, m_platform, m_sysuuid): + """Provisioning env with a metadata disk should return None.""" + m_platform.return_value = ( + ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh") + self.assertIsNone(ibm.read_md()) + + def test_provisioning_no_metadata(self, m_platform, m_sysuuid): + """Provisioning env with no metadata disk should return None.""" + m_platform.return_value = ( + ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None) + self.assertIsNone(ibm.read_md()) + + def test_provisioning_not_ibm(self, m_platform, m_sysuuid): + """Provisioning env but not identified as IBM should return None.""" + m_platform.return_value = (None, None) + self.assertIsNone(ibm.read_md()) + + def test_template_live(self, m_platform, m_sysuuid): + """Template live environment should be identified.""" + tmpdir = self.tmp_dir() + m_platform.return_value = ( + ibm.Platforms.TEMPLATE_LIVE_METADATA, tmpdir) + m_sysuuid.return_value = self.sysuuid + + test_helpers.populate_dir(tmpdir, { + 'openstack/latest/meta_data.json': json.dumps(self.template_md), + 'openstack/latest/user_data': self.userdata, + 'openstack/content/interfaces': self.content_interfaces, + 'meta.js': self.meta_js}) + + ret = ibm.read_md() + self.assertEqual(ibm.Platforms.TEMPLATE_LIVE_METADATA, + ret['platform']) + self.assertEqual(tmpdir, ret['source']) + self.assertEqual(self.userdata, ret['userdata']) + self.assertEqual(self._get_expected_metadata(self.template_md), + ret['metadata']) + self.assertEqual(self.sysuuid, ret['system-uuid']) + + def test_os_code_live(self, m_platform, m_sysuuid): + """Verify an os_code metadata path.""" + tmpdir = self.tmp_dir() + m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir) + netdata = json.dumps(self.network_data) + test_helpers.populate_dir(tmpdir, { + 'openstack/latest/meta_data.json': json.dumps(self.oscode_md), + 'openstack/latest/user_data': self.userdata, + 'openstack/latest/vendor_data.json': json.dumps(self.vendor_data), + 'openstack/latest/network_data.json': netdata, + }) + + ret = ibm.read_md() + self.assertEqual(ibm.Platforms.OS_CODE, ret['platform']) + self.assertEqual(tmpdir, ret['source']) + self.assertEqual(self.userdata, ret['userdata']) + self.assertEqual(self._get_expected_metadata(self.oscode_md), + ret['metadata']) + + def test_os_code_live_no_userdata(self, m_platform, m_sysuuid): + """Verify os_code without user-data.""" + tmpdir = self.tmp_dir() + m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir) + test_helpers.populate_dir(tmpdir, { + 'openstack/latest/meta_data.json': json.dumps(self.oscode_md), + 'openstack/latest/vendor_data.json': json.dumps(self.vendor_data), + }) + + ret = ibm.read_md() + self.assertEqual(ibm.Platforms.OS_CODE, ret['platform']) + self.assertEqual(tmpdir, ret['source']) + self.assertIsNone(ret['userdata']) + self.assertEqual(self._get_expected_metadata(self.oscode_md), + ret['metadata']) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 85999b7a..53643989 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -9,6 +9,8 @@ from cloudinit import util from cloudinit.tests.helpers import ( CiTestCase, dir2dict, populate_dir) +from cloudinit.sources import DataSourceIBMCloud as dsibm + UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") UNAME_PPC64EL = ("Linux diamond 4.4.0-83-generic #106-Ubuntu SMP " @@ -37,8 +39,8 @@ BLKID_UEFI_UBUNTU = [ POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled" POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled" -DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled" -DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=disabled" +DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=disabled" +DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=enabled" DI_EC2_STRICT_ID_DEFAULT = "true" OVF_MATCH_STRING = 'http://schemas.dmtf.org/ovf/environment/1' @@ -64,6 +66,9 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor" P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" +IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg" +IBM_CONFIG_UUID = "9796-932E" + MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} @@ -239,6 +244,57 @@ class TestDsIdentify(CiTestCase): self._test_ds_found('ConfigDriveUpper') return + def test_ibmcloud_template_userdata_in_provisioning(self): + """Template provisioned with user-data during provisioning stage. + + Template provisioning with user-data has METADATA disk, + datasource should return not found.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-metadata']) + data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + return self._check_via_dict(data, RC_NOT_FOUND) + + def test_ibmcloud_template_userdata(self): + """Template provisioned with user-data first boot. + + Template provisioning with user-data has METADATA disk. + datasource should return found.""" + self._test_ds_found('IBMCloud-metadata') + + def test_ibmcloud_template_no_userdata_in_provisioning(self): + """Template provisioned with no user-data during provisioning. + + no disks attached. Datasource should return not found.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks']) + data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + return self._check_via_dict(data, RC_NOT_FOUND) + + def test_ibmcloud_template_no_userdata(self): + """Template provisioned with no user-data first boot. + + no disks attached. Datasource should return found.""" + self._check_via_dict(VALID_CFG['IBMCloud-nodisks'], RC_NOT_FOUND) + + def test_ibmcloud_os_code(self): + """Launched by os code always has config-2 disk.""" + self._test_ds_found('IBMCloud-config-2') + + def test_ibmcloud_os_code_different_uuid(self): + """IBM cloud config-2 disks must be explicit match on UUID. + + If the UUID is not 9796-932E then we actually expect ConfigDrive.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + offset = None + for m, d in enumerate(data['mocks']): + if d.get('name') == "blkid": + offset = m + break + if not offset: + raise ValueError("Expected to find 'blkid' mock, but did not.") + data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID, + "DEAD-BEEF") + self._check_via_dict( + data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE]) + def test_policy_disabled(self): """A Builtin policy of 'disabled' should return not found. @@ -452,7 +508,7 @@ VALID_CFG = { }, 'Ec2-xen': { 'ds': 'Ec2', - 'mocks': [{'name': 'detect_virt', 'RET': 'xen', 'ret': 0}], + 'mocks': [MOCK_VIRT_IS_XEN], 'files': { 'sys/hypervisor/uuid': 'ec2c6e2f-5fac-4fc7-9c82-74127ec14bbb\n' }, @@ -579,6 +635,48 @@ VALID_CFG = { 'ds': 'Hetzner', 'files': {P_SYS_VENDOR: 'Hetzner\n'}, }, + 'IBMCloud-metadata': { + 'ds': 'IBMCloud', + 'mocks': [ + MOCK_VIRT_IS_XEN, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, + {'DEVNAME': 'xvda2', 'TYPE': 'ext4', + 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()}, + {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'METADATA'}]), + }, + ], + }, + 'IBMCloud-config-2': { + 'ds': 'IBMCloud', + 'mocks': [ + MOCK_VIRT_IS_XEN, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(), + 'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'}, + {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2', + 'UUID': dsibm.IBM_CONFIG_UUID}, + {'DEVNAME': 'xvda2', 'TYPE': 'ext4', + 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(), + 'UUID': uuid4()}, + ]), + }, + ], + }, + 'IBMCloud-nodisks': { + 'ds': 'IBMCloud', + 'mocks': [ + MOCK_VIRT_IS_XEN, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, + {'DEVNAME': 'xvda2', 'TYPE': 'ext4', + 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()}]), + }, + ], + }, } # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index e2552c8b..9a2db5c4 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -92,6 +92,7 @@ DI_DMI_SYS_VENDOR="" DI_DMI_PRODUCT_SERIAL="" DI_DMI_PRODUCT_UUID="" DI_FS_LABELS="" +DI_FS_UUIDS="" DI_ISO9660_DEVS="" DI_KERNEL_CMDLINE="" DI_VIRT="" @@ -114,7 +115,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner" +OVF SmartOS Scaleway Hetzner IBMCloud" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -123,6 +124,8 @@ DI_ON_NOTFOUND="" DI_EC2_STRICT_ID_DEFAULT="true" +_IS_IBM_CLOUD="" + error() { set -- "ERROR:" "$@"; debug 0 "$@" @@ -196,7 +199,7 @@ read_fs_info() { return fi local oifs="$IFS" line="" delim="," - local ret=0 out="" labels="" dev="" label="" ftype="" isodevs="" + local ret=0 out="" labels="" dev="" label="" ftype="" isodevs="" uuids="" out=$(blkid -c /dev/null -o export) || { ret=$? error "failed running [$ret]: blkid -c /dev/null -o export" @@ -219,12 +222,14 @@ read_fs_info() { LABEL=*) label="${line#LABEL=}"; labels="${labels}${line#LABEL=}${delim}";; TYPE=*) ftype=${line#TYPE=};; + UUID=*) uuids="${uuids}${line#UUID=}$delim";; esac done [ -n "$dev" -a "$ftype" = "iso9660" ] && isodevs="${isodevs} ${dev}=$label" DI_FS_LABELS="${labels%${delim}}" + DI_FS_UUIDS="${uuids%${delim}}" DI_ISO9660_DEVS="${isodevs# }" } @@ -437,14 +442,25 @@ dmi_sys_vendor_is() { [ "${DI_DMI_SYS_VENDOR}" = "$1" ] } -has_fs_with_label() { - local label="$1" - case ",${DI_FS_LABELS}," in - *,$label,*) return 0;; +has_fs_with_uuid() { + case ",${DI_FS_UUIDS}," in + *,$1,*) return 0;; esac return 1 } +has_fs_with_label() { + # has_fs_with_label(label1[ ,label2 ..]) + # return 0 if a there is a filesystem that matches any of the labels. + local label="" + for label in "$@"; do + case ",${DI_FS_LABELS}," in + *,$label,*) return 0;; + esac + done + return 1 +} + nocase_equal() { # nocase_equal(a, b) # return 0 if case insenstive comparision a.lower() == b.lower() @@ -583,6 +599,8 @@ dscheck_NoCloud() { case " ${DI_DMI_PRODUCT_SERIAL} " in *\ ds=nocloud*) return ${DS_FOUND};; esac + + is_ibm_cloud && return ${DS_NOT_FOUND} for d in nocloud nocloud-net; do check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND} @@ -594,9 +612,8 @@ dscheck_NoCloud() { } check_configdrive_v2() { - if has_fs_with_label "config-2"; then - return ${DS_FOUND} - elif has_fs_with_label "CONFIG-2"; then + is_ibm_cloud && return ${DS_NOT_FOUND} + if has_fs_with_label CONFIG-2 config-2; then return ${DS_FOUND} fi # look in /config-drive /seed/config_drive for a directory @@ -988,6 +1005,36 @@ dscheck_Hetzner() { return ${DS_NOT_FOUND} } +is_ibm_provisioning() { + [ -f "${PATH_ROOT}/root/provisioningConfiguration.cfg" ] +} + +is_ibm_cloud() { + cached "${_IS_IBM_CLOUD}" && return ${_IS_IBM_CLOUD} + local ret=1 + if [ "$DI_VIRT" = "xen" ]; then + if is_ibm_provisioning; then + ret=0 + elif has_fs_with_label METADATA metadata; then + ret=0 + elif has_fs_with_uuid 9796-932E && + has_fs_with_label CONFIG-2 config-2; then + ret=0 + fi + fi + _IS_IBM_CLOUD=$ret + return $ret +} + +dscheck_IBMCloud() { + if is_ibm_provisioning; then + debug 1 "cloud-init disabled during provisioning on IBMCloud" + return ${DS_NOT_FOUND} + fi + is_ibm_cloud && return ${DS_FOUND} + return ${DS_NOT_FOUND} +} + collect_info() { read_virt read_pid1_product_name -- cgit v1.2.3 From bdeec31dca5d9a8dc98ae31fc20a29557a2ec59e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 23 Mar 2018 20:39:21 -0600 Subject: Enable IBMCloud datasource in settings.py. This just got missed in the IBMCloud datasource addition. Add it to the builtin list of datasources. --- cloudinit/settings.py | 1 + tests/unittests/test_datasource/test_common.py | 2 ++ 2 files changed, 3 insertions(+) (limited to 'tests') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 5fe749d4..dde5749d 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -37,6 +37,7 @@ CFG_BUILTIN = { 'Bigstep', 'Scaleway', 'Hetzner', + 'IBMCloud', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 6d2dc5b5..ec333888 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -15,6 +15,7 @@ from cloudinit.sources import ( DataSourceEc2 as Ec2, DataSourceGCE as GCE, DataSourceHetzner as Hetzner, + DataSourceIBMCloud as IBMCloud, DataSourceMAAS as MAAS, DataSourceNoCloud as NoCloud, DataSourceOpenNebula as OpenNebula, @@ -33,6 +34,7 @@ DEFAULT_LOCAL = [ ConfigDrive.DataSourceConfigDrive, DigitalOcean.DataSourceDigitalOcean, Hetzner.DataSourceHetzner, + IBMCloud.DataSourceIBMCloud, NoCloud.DataSourceNoCloud, OpenNebula.DataSourceOpenNebula, OVF.DataSourceOVF, -- cgit v1.2.3 From 20e3ddab7f55c2bf5e700c69fd24a0ac2206dbcf Mon Sep 17 00:00:00 2001 From: Dominic Schlegel Date: Tue, 27 Mar 2018 10:10:42 -0400 Subject: FreeBSD: resizefs module now able to handle zfs/zpool. Previously there was no support at all for zfs file system. With this change it is now possible to use the resizefs module to grow a zpool to its maximum partition size on FreeBSD. LP: #1721243 --- cloudinit/config/cc_resizefs.py | 22 ++++++++ cloudinit/util.py | 44 ++++++++++++---- tests/data/mount_parse_ext.txt | 19 +++++++ tests/data/mount_parse_zfs.txt | 21 ++++++++ tests/data/zpool_status_simple.txt | 10 ++++ .../test_handler/test_handler_resizefs.py | 58 +++++++++++++++++++++- tests/unittests/test_util.py | 50 +++++++++++++++++++ 7 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 tests/data/mount_parse_ext.txt create mode 100644 tests/data/mount_parse_zfs.txt create mode 100644 tests/data/zpool_status_simple.txt (limited to 'tests') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cec22bb7..c8e1752f 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -84,6 +84,10 @@ def _resize_ufs(mount_point, devpth): return ('growfs', devpth) +def _resize_zfs(mount_point, devpth): + return ('zpool', 'online', '-e', mount_point, devpth) + + def _get_dumpfs_output(mount_point): dumpfs_res, err = util.subp(['dumpfs', '-m', mount_point]) return dumpfs_res @@ -148,6 +152,7 @@ RESIZE_FS_PREFIXES_CMDS = [ ('ext', _resize_ext), ('xfs', _resize_xfs), ('ufs', _resize_ufs), + ('zfs', _resize_zfs), ] RESIZE_FS_PRECHECK_CMDS = { @@ -188,6 +193,13 @@ def maybe_get_writable_device_path(devpath, info, log): log.debug("Not attempting to resize devpath '%s': %s", devpath, info) return None + # FreeBSD zpool can also just use gpt/