diff options
Diffstat (limited to 'tests')
100 files changed, 5780 insertions, 1684 deletions
diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py index c6c1877b..ab345491 100644 --- a/tests/cloud_tests/args.py +++ b/tests/cloud_tests/args.py @@ -62,6 +62,9 @@ ARG_SETS = { (('-d', '--data-dir'), {'help': 'directory to store test data in', 'action': 'store', 'metavar': 'DIR', 'required': False}), + (('--preserve-instance',), + {'help': 'do not destroy the instance under test', + 'action': 'store_true', 'default': False, 'required': False}), (('--preserve-data',), {'help': 'do not remove collected data after successful run', 'action': 'store_true', 'default': False, 'required': False}),), diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index b9cfcfa6..f04d0cd4 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -113,7 +113,7 @@ def bddeb(args): @return_value: fail count """ LOG.info('preparing to build cloud-init deb') - (res, failed) = run_stage('build deb', [partial(setup_build, args)]) + _res, failed = run_stage('build deb', [partial(setup_build, args)]) return failed # vi: ts=4 expandtab diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index d4f9135b..642745d8 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -9,6 +9,7 @@ from cloudinit import util as c_util from tests.cloud_tests import (config, LOG, setup_image, util) from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) from tests.cloud_tests import platforms +from tests.cloud_tests.testcases import base, get_test_class def collect_script(instance, base_dir, script, script_name): @@ -25,7 +26,8 @@ def collect_script(instance, base_dir, script, script_name): script.encode(), rcs=False, description='collect: {}'.format(script_name)) if err: - LOG.debug("collect script %s had stderr: %s", script_name, err) + LOG.debug("collect script %s exited '%s' and had stderr: %s", + script_name, err, exit) if not isinstance(out, bytes): raise util.PlatformError( "Collection of '%s' returned type %s, expected bytes: %s" % @@ -41,7 +43,7 @@ def collect_console(instance, base_dir): @param base_dir: directory to write console log to """ logfile = os.path.join(base_dir, 'console.log') - LOG.debug('getting console log for %s to %s', instance, logfile) + LOG.debug('getting console log for %s to %s', instance.name, logfile) try: data = instance.console_log() except NotImplementedError as e: @@ -62,6 +64,7 @@ def collect_test_data(args, snapshot, os_name, test_name): res = ({}, 1) # load test config + test_name_in = test_name test_name = config.path_to_name(test_name) test_config = config.load_test_config(test_name) user_data = test_config['cloud_config'] @@ -74,6 +77,16 @@ def collect_test_data(args, snapshot, os_name, test_name): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) + test_class = get_test_class( + config.name_to_module(test_name_in), + test_data={'platform': snapshot.platform_name, 'os_name': os_name}, + test_conf=test_config['cloud_config']) + try: + test_class.maybeSkipTest() + except base.SkipTest as s: + LOG.warning('skipping test config %s: %s', test_name, s) + return ({}, 0) + # if testcase requires a feature flag that the image does not support, # skip the testcase with a warning req_features = test_config.get('required_features', []) @@ -92,7 +105,8 @@ def collect_test_data(args, snapshot, os_name, test_name): # create test instance component = PlatformComponent( partial(platforms.get_instance, snapshot, user_data, - block=True, start=False, use_desc=test_name)) + block=True, start=False, use_desc=test_name), + preserve_instance=args.preserve_instance) LOG.info('collecting test data for test: %s', test_name) with component as instance: diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index 3bad021f..529e79cd 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -87,32 +87,39 @@ class Instance(TargetBase): self._ssh_client = None def _ssh_connect(self): - """Connect via SSH.""" + """Connect via SSH. + + Attempt to SSH to the client on the specific IP and port. If it + fails in some manner, then retry 2 more times for a total of 3 + attempts; sleeping a few seconds between attempts. + """ if self._ssh_client: return self._ssh_client if not self.ssh_ip or not self.ssh_port: - raise ValueError + raise ValueError("Cannot ssh_connect, ssh_ip=%s ssh_port=%s" % + (self.ssh_ip, self.ssh_port)) client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - retries = 30 + retries = 3 while retries: try: client.connect(username=self.ssh_username, hostname=self.ssh_ip, port=self.ssh_port, - pkey=private_key, banner_timeout=30) + pkey=private_key) self._ssh_client = client return client except (ConnectionRefusedError, AuthenticationException, BadHostKeyException, ConnectionResetError, SSHException, - OSError) as e: + OSError): retries -= 1 - time.sleep(10) + LOG.debug('Retrying ssh connection on connect failure') + time.sleep(3) - ssh_cmd = 'Failed ssh connection to %s@%s:%s after 300 seconds' % ( + ssh_cmd = 'Failed ssh connection to %s@%s:%s after 3 retries' % ( self.ssh_username, self.ssh_ip, self.ssh_port ) raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') @@ -128,18 +135,31 @@ class Instance(TargetBase): return ' '.join(l for l in test.strip().splitlines() if not l.lstrip().startswith('#')) - time = self.config['boot_timeout'] + boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] if wait_for_cloud_init: tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' - 'exit 0; sleep 1; done; exit 1').format(time=time, + 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, test=formatted_tests) - if self.execute(cmd, rcs=(0, 1))[-1] != 0: - raise OSError('timeout: after {}s system not started'.format(time)) - + end_time = time.time() + boot_timeout + while True: + try: + return_code = self.execute( + cmd, rcs=(0, 1), description='wait for instance start' + )[-1] + if return_code == 0: + break + except util.InTargetExecuteError: + LOG.warning("failed to connect via SSH") + + if time.time() < end_time: + time.sleep(3) + else: + raise util.PlatformError('ssh', 'after %ss instance is not ' + 'reachable' % boot_timeout) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 0d957bca..83c97ab4 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -12,6 +12,8 @@ from tests.cloud_tests.util import PlatformError from ..instances import Instance +from pylxd import exceptions as pylxd_exc + class LXDInstance(Instance): """LXD container backed instance.""" @@ -30,6 +32,9 @@ class LXDInstance(Instance): @param config: image config @param features: supported feature flags """ + if not pylxd_container: + raise ValueError("Invalid value pylxd_container: %s" % + pylxd_container) self._pylxd_container = pylxd_container super(LXDInstance, self).__init__( platform, name, properties, config, features) @@ -40,9 +45,19 @@ class LXDInstance(Instance): @property def pylxd_container(self): """Property function.""" + if self._pylxd_container is None: + raise RuntimeError( + "%s: Attempted use of pylxd_container after deletion." % self) self._pylxd_container.sync() return self._pylxd_container + def __str__(self): + return ( + '%s(name=%s) status=%s' % + (self.__class__.__name__, self.name, + ("deleted" if self._pylxd_container is None else + self.pylxd_container.status))) + def _execute(self, command, stdin=None, env=None): if env is None: env = {} @@ -152,9 +167,8 @@ class LXDInstance(Instance): return fp.read() try: - stdout, stderr = subp( - ['lxc', 'console', '--show-log', self.name], decode=False) - return stdout + return subp(['lxc', 'console', '--show-log', self.name], + decode=False)[0] except ProcessExecutionError as e: raise PlatformError( "console log", @@ -166,10 +180,27 @@ class LXDInstance(Instance): self.shutdown(wait=wait) self.start(wait=wait) - def shutdown(self, wait=True): + def shutdown(self, wait=True, retry=1): """Shutdown instance.""" - if self.pylxd_container.status != 'Stopped': + if self.pylxd_container.status == 'Stopped': + return + + try: + LOG.debug("%s: shutting down (wait=%s)", self, wait) self.pylxd_container.stop(wait=wait) + except (pylxd_exc.LXDAPIException, pylxd_exc.NotFound) as e: + # An exception happens here sometimes (LP: #1783198) + # LOG it, and try again. + LOG.warning( + ("%s: shutdown(retry=%d) caught %s in shutdown " + "(response=%s): %s"), + self, retry, e.__class__.__name__, e.response, e) + if isinstance(e, pylxd_exc.NotFound): + LOG.debug("container_exists(%s) == %s", + self.name, self.platform.container_exists(self.name)) + if retry == 0: + raise e + return self.shutdown(wait=wait, retry=retry - 1) def start(self, wait=True, wait_for_cloud_init=False): """Start instance.""" @@ -190,12 +221,14 @@ class LXDInstance(Instance): def destroy(self): """Clean up instance.""" + LOG.debug("%s: deleting container.", self) self.unfreeze() self.shutdown() self.pylxd_container.delete(wait=True) + self._pylxd_container = None + if self.platform.container_exists(self.name): - raise OSError('container {} was not properly removed' - .format(self.name)) + raise OSError('%s: container was not properly removed' % self) if self._console_log_file and os.path.exists(self._console_log_file): os.unlink(self._console_log_file) shutil.rmtree(self.tmpd) @@ -209,16 +242,15 @@ def _has_proper_console_support(): if 'console' not in info.get('api_extensions', []): reason = "LXD server does not support console api extension" else: - dver = info.get('environment', {}).get('driver_version', "") + dver = str(info.get('environment', {}).get('driver_version', "")) if dver.startswith("2.") or dver.startswith("1."): reason = "LXD Driver version not 3.x+ (%s)" % dver else: try: - stdout, stderr = subp(['lxc', 'console', '--help'], - decode=False) + stdout = subp(['lxc', 'console', '--help'], decode=False)[0] if not (b'console' in stdout and b'log' in stdout): reason = "no '--log' in lxc console --help" - except ProcessExecutionError as e: + except ProcessExecutionError: reason = "no 'console' command in lxc client" if reason: diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index c7dcbe83..defae02b 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -129,6 +129,22 @@ features: releases: # UBUNTU ================================================================= + cosmic: + # EOL: Jul 2019 + default: + enabled: true + release: cosmic + version: 18.10 + os: ubuntu + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: cosmic + setup_overrides: null + override_templates: false bionic: # EOL: Apr 2023 default: diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 6d242115..39f4517f 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -4,6 +4,7 @@ from functools import partial import os +import yaml from tests.cloud_tests import LOG from tests.cloud_tests import stage, util @@ -25,10 +26,9 @@ def installed_package_version(image, package, ensure_installed=True): else: raise NotImplementedError - msg = 'query version for package: {}'.format(package) - (out, err, exit) = image.execute( - cmd, description=msg, rcs=(0,) if ensure_installed else range(0, 256)) - return out.strip() + return image.execute( + cmd, description='query version for package: {}'.format(package), + rcs=(0,) if ensure_installed else range(0, 256))[0].strip() def install_deb(args, image): @@ -54,7 +54,7 @@ def install_deb(args, image): remote_path], description=msg) # check installed deb version matches package fmt = ['-W', "--showformat=${Version}"] - (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) + out = image.execute(['dpkg-deb'] + fmt + [remote_path])[0] expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: @@ -85,7 +85,7 @@ def install_rpm(args, image): image.execute(['rpm', '-U', remote_path], description=msg) fmt = ['--queryformat', '"%{VERSION}"'] - (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) + (out, _err, _exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: @@ -221,7 +221,14 @@ def setup_image(args, image): calls = [partial(stage.run_single, desc, partial(func, args, image)) for name, func, desc in handlers if getattr(args, name, None)] - LOG.info('setting up %s', image) + try: + data = yaml.load(image.read_data("/etc/cloud/build.info", decode=True)) + info = ' '.join(["%s=%s" % (k, data.get(k)) + for k in ("build_name", "serial") if k in data]) + except Exception as e: + info = "N/A (%s)" % e + + LOG.info('setting up %s (%s)', image, info) res = stage.run_stage( 'set up for {}'.format(image), calls, continue_after_error=False) return res diff --git a/tests/cloud_tests/stage.py b/tests/cloud_tests/stage.py index 74a7d46d..d64a1dcc 100644 --- a/tests/cloud_tests/stage.py +++ b/tests/cloud_tests/stage.py @@ -12,9 +12,15 @@ from tests.cloud_tests import LOG class PlatformComponent(object): """Context manager to safely handle platform components.""" - def __init__(self, get_func): - """Store get_<platform component> function as partial with no args.""" + def __init__(self, get_func, preserve_instance=False): + """Store get_<platform component> function as partial with no args. + + @param get_func: Callable returning an instance from the platform. + @param preserve_instance: Boolean, when True, do not destroy instance + after test. Used for test development. + """ self.get_func = get_func + self.preserve_instance = preserve_instance def __enter__(self): """Create instance of platform component.""" @@ -24,7 +30,10 @@ class PlatformComponent(object): def __exit__(self, etype, value, trace): """Destroy instance.""" if self.instance is not None: - self.instance.destroy() + if self.preserve_instance: + LOG.info('Preserving test instance %s', self.instance.name) + else: + self.instance.destroy() def run_single(name, call): diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index a3e29900..fb9a5d27 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -24,9 +24,13 @@ base_test_data: status.json: | #!/bin/sh cat /run/cloud-init/status.json - cloud-init-version: | + package-versions: | #!/bin/sh - dpkg-query -W -f='${Version}' cloud-init + dpkg-query --show + build.info: | + #!/bin/sh + binfo=/etc/cloud/build.info + [ -f "$binfo" ] && cat "$binfo" || echo "N/A" system.journal.gz: | #!/bin/sh [ -d /run/systemd ] || { echo "not systemd."; exit 0; } diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index bd548f5a..6bb39f77 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,8 +4,7 @@ import importlib import inspect -import unittest -from unittest.util import strclass +import unittest2 from cloudinit.util import read_conf @@ -13,7 +12,7 @@ from tests.cloud_tests import config from tests.cloud_tests.testcases.base import CloudTestCase as base_test -def discover_tests(test_name): +def discover_test(test_name): """Discover tests in test file for 'testname'. @return_value: list of test classes @@ -25,35 +24,48 @@ def discover_tests(test_name): except NameError: raise ValueError('no test verifier found at: {}'.format(testmod_name)) - return [mod for name, mod in inspect.getmembers(testmod) - if inspect.isclass(mod) and base_test in inspect.getmro(mod) and - getattr(mod, '__test__', True)] + found = [mod for name, mod in inspect.getmembers(testmod) + if (inspect.isclass(mod) + and base_test in inspect.getmro(mod) + and getattr(mod, '__test__', True))] + if len(found) != 1: + raise RuntimeError( + "Unexpected situation, multiple tests for %s: %s" % ( + test_name, found)) + return found -def get_suite(test_name, data, conf): - """Get test suite with all tests for 'testname'. - @return_value: a test suite - """ - suite = unittest.TestSuite() - for test_class in discover_tests(test_name): +def get_test_class(test_name, test_data, test_conf): + test_class = discover_test(test_name)[0] + + class DynamicTestSubclass(test_class): - class tmp(test_class): + _realclass = test_class + data = test_data + conf = test_conf + release_conf = read_conf(config.RELEASES_CONF)['releases'] - _realclass = test_class + def __str__(self): + return "%s (%s)" % (self._testMethodName, + unittest2.util.strclass(self._realclass)) - def __str__(self): - return "%s (%s)" % (self._testMethodName, - strclass(self._realclass)) + @classmethod + def setUpClass(cls): + cls.maybeSkipTest() - @classmethod - def setUpClass(cls): - cls.data = data - cls.conf = conf - cls.release_conf = read_conf(config.RELEASES_CONF)['releases'] + return DynamicTestSubclass - suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp)) +def get_suite(test_name, data, conf): + """Get test suite with all tests for 'testname'. + + @return_value: a test suite + """ + suite = unittest2.TestSuite() + suite.addTest( + unittest2.defaultTestLoader.loadTestsFromTestCase( + get_test_class(test_name, data, conf))) return suite # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 324c7c91..e18d601c 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -5,15 +5,15 @@ import crypt import json import re -import unittest +import unittest2 from cloudinit import util as c_util -SkipTest = unittest.SkipTest +SkipTest = unittest2.SkipTest -class CloudTestCase(unittest.TestCase): +class CloudTestCase(unittest2.TestCase): """Base test class for verifiers.""" # data gets populated in get_suite.setUpClass @@ -31,6 +31,32 @@ class CloudTestCase(unittest.TestCase): def is_distro(self, distro_name): return self.os_cfg['os'] == distro_name + @classmethod + def maybeSkipTest(cls): + """Present to allow subclasses to override and raise a skipTest.""" + pass + + def assertPackageInstalled(self, name, version=None): + """Check dpkg-query --show output for matching package name. + + @param name: package base name + @param version: string representing a package version or part of a + version. + """ + pkg_out = self.get_data_file('package-versions') + pkg_match = re.search( + '^%s\t(?P<version>.*)$' % name, pkg_out, re.MULTILINE) + if pkg_match: + installed_version = pkg_match.group('version') + if not version: + return # Success + if installed_version.startswith(version): + return # Success + raise AssertionError( + 'Expected package version %s-%s not found. Found %s' % + name, version, installed_version) + raise AssertionError('Package not installed: %s' % name) + def os_version_cmp(self, cmp_version): """Compare the version of the test to comparison_version. @@ -146,29 +172,31 @@ class CloudTestCase(unittest.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertEqual( - ['ds/user-data'], instance_data['base64-encoded-keys']) + self.assertItemsEqual( + [], + instance_data['base64_encoded_keys']) ds = instance_data.get('ds', {}) - macs = ds.get('network', {}).get('interfaces', {}).get('macs', {}) + v1_data = instance_data.get('v1', {}) + metadata = ds.get('meta_data', {}) + macs = metadata.get( + 'network', {}).get('interfaces', {}).get('macs', {}) if not macs: raise AssertionError('No network data from EC2 meta-data') # Check meta-data items we depend on expected_net_keys = [ 'public-ipv4s', 'ipv4-associations', 'local-hostname', 'public-hostname'] - for mac, mac_data in macs.items(): + for mac_data in macs.values(): for key in expected_net_keys: self.assertIn(key, mac_data) self.assertIsNotNone( - ds.get('placement', {}).get('availability-zone'), + metadata.get('placement', {}).get('availability-zone'), 'Could not determine EC2 Availability zone placement') - ds = instance_data.get('ds', {}) - v1_data = instance_data.get('v1', {}) self.assertIsNotNone( - v1_data['availability-zone'], 'expected ec2 availability-zone') - self.assertEqual('aws', v1_data['cloud-name']) - self.assertIn('i-', v1_data['instance-id']) - self.assertIn('ip-', v1_data['local-hostname']) + v1_data['availability_zone'], 'expected ec2 availability_zone') + self.assertEqual('aws', v1_data['cloud_name']) + self.assertIn('i-', v1_data['instance_id']) + self.assertIn('ip-', v1_data['local_hostname']) self.assertIsNotNone(v1_data['region'], 'expected ec2 region') def test_instance_data_json_lxd(self): @@ -191,16 +219,14 @@ class CloudTestCase(unittest.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertEqual( - ['ds/user-data', 'ds/vendor-data'], - sorted(instance_data['base64-encoded-keys'])) - self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys'])) + self.assertEqual('nocloud', v1_data['cloud_name']) self.assertIsNone( - v1_data['availability-zone'], - 'found unexpected lxd availability-zone %s' % - v1_data['availability-zone']) - self.assertIn('cloud-test', v1_data['instance-id']) - self.assertIn('cloud-test', v1_data['local-hostname']) + v1_data['availability_zone'], + 'found unexpected lxd availability_zone %s' % + v1_data['availability_zone']) + self.assertIn('cloud-test', v1_data['instance_id']) + self.assertIn('cloud-test', v1_data['local_hostname']) self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) @@ -226,18 +252,17 @@ class CloudTestCase(unittest.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertEqual( - ['ds/user-data'], instance_data['base64-encoded-keys']) - self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertItemsEqual([], instance_data['base64_encoded_keys']) + self.assertEqual('nocloud', v1_data['cloud_name']) self.assertIsNone( - v1_data['availability-zone'], - 'found unexpected kvm availability-zone %s' % - v1_data['availability-zone']) + v1_data['availability_zone'], + 'found unexpected kvm availability_zone %s' % + v1_data['availability_zone']) self.assertIsNotNone( - re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', - v1_data['instance-id']), - 'kvm instance-id is not a UUID: %s' % v1_data['instance-id']) - self.assertIn('ubuntu', v1_data['local-hostname']) + re.match(r'[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', + v1_data['instance_id']), + 'kvm instance_id is not a UUID: %s' % v1_data['instance_id']) + self.assertIn('ubuntu', v1_data['local_hostname']) self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.py +++ b/tests/cloud_tests/testcases/examples/including_user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/testcases/modules/byobu.py b/tests/cloud_tests/testcases/modules/byobu.py index 005ca014..74d0529a 100644 --- a/tests/cloud_tests/testcases/modules/byobu.py +++ b/tests/cloud_tests/testcases/modules/byobu.py @@ -9,8 +9,7 @@ class TestByobu(base.CloudTestCase): def test_byobu_installed(self): """Test byobu installed.""" - out = self.get_data_file('byobu_installed') - self.assertIn('/usr/bin/byobu', out) + self.assertPackageInstalled('byobu') def test_byobu_profile_enabled(self): """Test byobu profile.d file exists.""" diff --git a/tests/cloud_tests/testcases/modules/byobu.yaml b/tests/cloud_tests/testcases/modules/byobu.yaml index a9aa1f3f..d002a611 100644 --- a/tests/cloud_tests/testcases/modules/byobu.yaml +++ b/tests/cloud_tests/testcases/modules/byobu.yaml @@ -7,9 +7,6 @@ cloud_config: | #cloud-config byobu_by_default: enable collect_scripts: - byobu_installed: | - #!/bin/bash - which byobu byobu_profile_enabled: | #!/bin/bash ls /etc/profile.d/Z97-byobu.sh diff --git a/tests/cloud_tests/testcases/modules/ca_certs.py b/tests/cloud_tests/testcases/modules/ca_certs.py index e75f0413..6b56f639 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.py +++ b/tests/cloud_tests/testcases/modules/ca_certs.py @@ -7,10 +7,23 @@ from tests.cloud_tests.testcases import base class TestCaCerts(base.CloudTestCase): """Test ca certs module.""" - def test_cert_count(self): - """Test the count is proper.""" - out = self.get_data_file('cert_count') - self.assertEqual(5, int(out)) + def test_certs_updated(self): + """Test certs have been updated in /etc/ssl/certs.""" + out = self.get_data_file('cert_links') + # Bionic update-ca-certificates creates less links debian #895075 + unlinked_files = [] + links = {} + for cert_line in out.splitlines(): + if '->' in cert_line: + fname, _sep, link = cert_line.split() + links[fname] = link + else: + unlinked_files.append(cert_line) + self.assertEqual(['ca-certificates.crt'], unlinked_files) + self.assertEqual('cloud-init-ca-certs.pem', links['a535c1f3.0']) + self.assertEqual( + '/usr/share/ca-certificates/cloud-init-ca-certs.crt', + links['cloud-init-ca-certs.pem']) def test_cert_installed(self): """Test line from our cert exists.""" diff --git a/tests/cloud_tests/testcases/modules/ca_certs.yaml b/tests/cloud_tests/testcases/modules/ca_certs.yaml index d939f435..2cd91551 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.yaml +++ b/tests/cloud_tests/testcases/modules/ca_certs.yaml @@ -43,9 +43,13 @@ cloud_config: | DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== -----END CERTIFICATE----- collect_scripts: - cert_count: | + cert_links: | #!/bin/bash - ls -l /etc/ssl/certs | wc -l + # links printed <filename> -> <link target> + # non-links printed <filename> + for file in `ls /etc/ssl/certs`; do + [ -h /etc/ssl/certs/$file ] && echo -n $file ' -> ' && readlink /etc/ssl/certs/$file || echo $file; + done cert: | #!/bin/bash md5sum /etc/ssl/certs/ca-certificates.crt diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py index c0262ba3..ea545e0a 100644 --- a/tests/cloud_tests/testcases/modules/lxd_bridge.py +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py @@ -7,15 +7,25 @@ from tests.cloud_tests.testcases import base class TestLxdBridge(base.CloudTestCase): """Test LXD module.""" + @classmethod + def maybeSkipTest(cls): + """Skip on cosmic for two reasons: + a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. + b.) apt install lxd installs via snap which can be slow + as that will download core snap and lxd.""" + os_name = cls.data.get('os_name', 'UNKNOWN') + if os_name == "cosmic": + raise base.SkipTest('Skipping test on cosmic (LP: #1795036).') + def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') - self.assertIn('/usr/bin/lxd', out) + self.assertIn('/lxd', out) def test_lxc(self): """Test lxc installed.""" out = self.get_data_file('lxc') - self.assertIn('/usr/bin/lxc', out) + self.assertIn('/lxc', out) def test_bridge(self): """Test bridge config.""" diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py index 1495674e..797bafed 100644 --- a/tests/cloud_tests/testcases/modules/lxd_dir.py +++ b/tests/cloud_tests/testcases/modules/lxd_dir.py @@ -7,14 +7,24 @@ from tests.cloud_tests.testcases import base class TestLxdDir(base.CloudTestCase): """Test LXD module.""" + @classmethod + def maybeSkipTest(cls): + """Skip on cosmic for two reasons: + a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. + b.) apt install lxd installs via snap which can be slow + as that will download core snap and lxd.""" + os_name = cls.data.get('os_name', 'UNKNOWN') + if os_name == "cosmic": + raise base.SkipTest('Skipping test on cosmic (LP: #1795036).') + def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') - self.assertIn('/usr/bin/lxd', out) + self.assertIn('/lxd', out) def test_lxc(self): """Test lxc installed.""" out = self.get_data_file('lxc') - self.assertIn('/usr/bin/lxc', out) + self.assertIn('/lxc', out) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp.py b/tests/cloud_tests/testcases/modules/ntp.py index b50e52fe..c63cc15e 100644 --- a/tests/cloud_tests/testcases/modules/ntp.py +++ b/tests/cloud_tests/testcases/modules/ntp.py @@ -9,15 +9,14 @@ class TestNtp(base.CloudTestCase): def test_ntp_installed(self): """Test ntp installed""" - out = self.get_data_file('ntp_installed') - self.assertEqual(0, int(out)) + self.assertPackageInstalled('ntp') def test_ntp_dist_entries(self): """Test dist config file is empty""" out = self.get_data_file('ntp_conf_dist_empty') self.assertEqual(0, int(out)) - def test_ntp_entires(self): + def test_ntp_entries(self): """Test config entries""" out = self.get_data_file('ntp_conf_pool_list') self.assertIn('pool.ntp.org iburst', out) diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml index 2530d72e..7ea0707d 100644 --- a/tests/cloud_tests/testcases/modules/ntp.yaml +++ b/tests/cloud_tests/testcases/modules/ntp.yaml @@ -4,6 +4,7 @@ cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: [] servers: [] collect_scripts: diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py new file mode 100644 index 00000000..0f4c3d08 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -0,0 +1,26 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +import unittest2 + +from tests.cloud_tests.testcases import base + + +class TestNtpChrony(base.CloudTestCase): + """Test ntp module with chrony client""" + + def setUp(self): + """Skip this suite of tests on lxd and artful or older.""" + if self.platform == 'lxd': + if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: + raise unittest2.SkipTest( + 'No support for chrony on containers <= artful.' + ' LP: #1589780') + return super(TestNtpChrony, self).setUp() + + def test_chrony_entries(self): + """Test chrony config entries""" + out = self.get_data_file('chrony_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.yaml b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml new file mode 100644 index 00000000..120735e2 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml @@ -0,0 +1,17 @@ +# +# ntp enabled, chrony selected, check conf file +# as chrony won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: chrony +collect_scripts: + chrony_conf: | + #!/bin/sh + set -- /etc/chrony.conf /etc/chrony/chrony.conf + for p in "$@"; do + [ -e "$p" ] && { cat "$p"; exit; } + done +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml index d490b228..60fa0fd1 100644 --- a/tests/cloud_tests/testcases/modules/ntp_pools.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml @@ -9,6 +9,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: - 0.cloud-init.mypool - 1.cloud-init.mypool diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml index 6b13b70e..ee636679 100644 --- a/tests/cloud_tests/testcases/modules/ntp_servers.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml @@ -6,6 +6,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp servers: - 172.16.15.14 - 172.16.17.18 diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.py b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py new file mode 100644 index 00000000..eca750bc --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py @@ -0,0 +1,15 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +from tests.cloud_tests.testcases import base + + +class TestNtpTimesyncd(base.CloudTestCase): + """Test ntp module with systemd-timesyncd client""" + + def test_timesyncd_entries(self): + """Test timesyncd config entries""" + out = self.get_data_file('timesyncd_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml new file mode 100644 index 00000000..ee47a741 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml @@ -0,0 +1,15 @@ +# +# ntp enabled, systemd-timesyncd selected, check conf file +# as systemd-timesyncd won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: systemd-timesyncd +collect_scripts: + timesyncd_conf: | + #!/bin/sh + cat /etc/systemd/timesyncd.conf.d/cloud-init.conf + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py index a92dec22..fecad768 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py @@ -7,15 +7,13 @@ from tests.cloud_tests.testcases import base class TestPackageInstallUpdateUpgrade(base.CloudTestCase): """Test package install update upgrade module.""" - def test_installed_htop(self): - """Test htop got installed.""" - out = self.get_data_file('dpkg_htop') - self.assertEqual(1, int(out)) + def test_installed_sl(self): + """Test sl got installed.""" + self.assertPackageInstalled('sl') def test_installed_tree(self): """Test tree got installed.""" - out = self.get_data_file('dpkg_tree') - self.assertEqual(1, int(out)) + self.assertPackageInstalled('tree') def test_apt_history(self): """Test apt history for update command.""" @@ -23,13 +21,13 @@ class TestPackageInstallUpdateUpgrade(base.CloudTestCase): self.assertIn( 'Commandline: /usr/bin/apt-get --option=Dpkg::Options' '::=--force-confold --option=Dpkg::options::=--force-unsafe-io ' - '--assume-yes --quiet install htop tree', out) + '--assume-yes --quiet install sl tree', out) def test_cloud_init_output(self): """Test cloud-init-output for install & upgrade stuff.""" out = self.get_data_file('cloud-init-output.log') self.assertIn('Setting up tree (', out) - self.assertIn('Setting up htop (', out) + self.assertIn('Setting up sl (', out) self.assertIn('Reading package lists...', out) self.assertIn('Building dependency tree...', out) self.assertIn('Reading state information...', out) diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml index 71d24b83..dd79e438 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml @@ -15,7 +15,7 @@ required_features: cloud_config: | #cloud-config packages: - - htop + - sl - tree package_update: true package_upgrade: true @@ -23,11 +23,8 @@ collect_scripts: apt_history_cmdline: | #!/bin/bash grep ^Commandline: /var/log/apt/history.log - dpkg_htop: | + dpkg_show: | #!/bin/bash - dpkg -l | grep htop | wc -l - dpkg_tree: | - #!/bin/bash - dpkg -l | grep tree | wc -l + dpkg-query --show # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py deleted file mode 100644 index 70917a4c..00000000 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ /dev/null @@ -1,39 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""cloud-init Integration Test Verify Script.""" -from tests.cloud_tests.testcases import base - - -class Test(base.CloudTestCase): - """Test salt minion module.""" - - def test_minon_master(self): - """Test master value in config.""" - out = self.get_data_file('minion') - self.assertIn('master: salt.mydomain.com', out) - - def test_minion_pem(self): - """Test private key.""" - out = self.get_data_file('minion.pem') - self.assertIn('------BEGIN PRIVATE KEY------', out) - self.assertIn('<key data>', out) - self.assertIn('------END PRIVATE KEY-------', out) - - def test_minion_pub(self): - """Test public key.""" - out = self.get_data_file('minion.pub') - self.assertIn('------BEGIN PUBLIC KEY-------', out) - self.assertIn('<key data>', out) - self.assertIn('------END PUBLIC KEY-------', out) - - def test_grains(self): - """Test master value in config.""" - out = self.get_data_file('grains') - self.assertIn('role: web', out) - - def test_minion_installed(self): - """Test if the salt-minion package is installed""" - out = self.get_data_file('minion_installed') - self.assertEqual(1, int(out)) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml deleted file mode 100644 index f20b9765..00000000 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Create config for a salt minion -# -# 2016-11-17: Currently takes >60 seconds results in test failure -# -enabled: True -cloud_config: | - #cloud-config - salt_minion: - conf: - master: salt.mydomain.com - public_key: | - ------BEGIN PUBLIC KEY------- - <key data> - ------END PUBLIC KEY------- - private_key: | - ------BEGIN PRIVATE KEY------ - <key data> - ------END PRIVATE KEY------- - grains: - role: web -collect_scripts: - minion: | - #!/bin/bash - cat /etc/salt/minion - minion_id: | - #!/bin/bash - cat /etc/salt/minion_id - minion.pem: | - #!/bin/bash - cat /etc/salt/pki/minion/minion.pem - minion.pub: | - #!/bin/bash - cat /etc/salt/pki/minion/minion.pub - grains: | - #!/bin/bash - cat /etc/salt/grains - minion_installed: | - #!/bin/bash - dpkg -l | grep salt-minion | grep ii | wc -l - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snap.yaml b/tests/cloud_tests/testcases/modules/snap.yaml index 44043f31..322199c3 100644 --- a/tests/cloud_tests/testcases/modules/snap.yaml +++ b/tests/cloud_tests/testcases/modules/snap.yaml @@ -1,6 +1,9 @@ # # Install snappy # +# Aug 23, 2018: Disabled due to requiring a proxy for testing +# tests do not handle the proxy well at this time. +enabled: False required_features: - snap cloud_config: | diff --git a/tests/cloud_tests/testcases/modules/snappy.yaml b/tests/cloud_tests/testcases/modules/snappy.yaml index 43f93295..8ac322ae 100644 --- a/tests/cloud_tests/testcases/modules/snappy.yaml +++ b/tests/cloud_tests/testcases/modules/snappy.yaml @@ -1,6 +1,9 @@ # # Install snappy # +# Aug 17, 2018: Disabled due to requiring a proxy for testing +# tests do not handle the proxy well at this time. +enabled: False required_features: - snap cloud_config: | diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.py +++ b/tests/cloud_tests/testcases/modules/user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/testcases/modules/write_files.py b/tests/cloud_tests/testcases/modules/write_files.py index 7bd520f6..526a2ebd 100644 --- a/tests/cloud_tests/testcases/modules/write_files.py +++ b/tests/cloud_tests/testcases/modules/write_files.py @@ -14,8 +14,11 @@ class TestWriteFiles(base.CloudTestCase): def test_binary(self): """Test binary file reads as executable.""" - out = self.get_data_file('file_binary') - self.assertIn('ELF 64-bit LSB executable, x86-64, version 1', out) + out = self.get_data_file('file_binary').strip() + md5 = "3801184b97bb8c6e63fa0e1eae2920d7" + sha256 = ("2c791c4037ea5bd7e928d6a87380f8ba7a803cd83d" + "5e4f269e28f5090f0f2c9a") + self.assertIn(out, (md5 + " -", sha256 + " -")) def test_gzip(self): """Test gzip file shows up as a shell script.""" diff --git a/tests/cloud_tests/testcases/modules/write_files.yaml b/tests/cloud_tests/testcases/modules/write_files.yaml index ce936b7b..cc7ea4bd 100644 --- a/tests/cloud_tests/testcases/modules/write_files.yaml +++ b/tests/cloud_tests/testcases/modules/write_files.yaml @@ -3,6 +3,13 @@ # # NOTE: on trusty 'file' has an output formatting error for binary files and # has 2 spaces in 'LSB executable', which causes a failure here +# +# NOTE: the binary data can be any binary data, not only executables +# and can be generated via the base 64 command as such: +# $ base64 < hello > hello.txt +# the opposite is running: +# $ base64 -d < hello.txt > hello +# required_features: - no_file_fmt_e cloud_config: | @@ -19,9 +26,7 @@ cloud_config: | SMBDOPTIONS="-D" path: /root/file_text - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA + /Z/xrHR4WINT0UNoKPQKbuovp6+Js+JK path: /root/file_binary permissions: '0555' - encoding: gzip @@ -38,7 +43,9 @@ collect_scripts: file /root/file_text file_binary: | #!/bin/bash - file /root/file_binary + for hasher in md5sum sha256sum; do + $hasher </root/file_binary && break + done file_gzip: | #!/bin/bash file /root/file_gzip diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 3dd4996d..06f7d865 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -358,7 +358,7 @@ class TargetBase(object): # when sh is invoked with '-c', then the first argument is "$0" # which is commonly understood as the "program name". # 'read_data' is the program name, and 'remote_path' is '$1' - stdout, stderr, rc = self._execute( + stdout, _stderr, rc = self._execute( ["sh", "-c", 'exec cat "$1"', 'read_data', remote_path]) if rc != 0: raise RuntimeError("Failed to read file '%s'" % remote_path) diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 5a68a484..9911ecf2 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest +import unittest2 from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {<test_name>: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) @@ -56,6 +56,51 @@ def verify_data(data_dir, platform, os_name, tests): return res +def format_test_failures(test_result): + """Return a human-readable printable format of test failures.""" + if not test_result['failures']: + return '' + failure_hdr = ' test failures:' + failure_fmt = ' * {module}.{class}.{function}\n {error}' + output = [] + for failure in test_result['failures']: + if not output: + output = [failure_hdr] + output.append(failure_fmt.format(**failure)) + return '\n'.join(output) + + +def format_results(res): + """Return human-readable results as a string""" + platform_hdr = 'Platform: {platform}' + distro_hdr = ' Distro: {distro}' + distro_summary_fmt = ( + ' test modules passed:{passed} tests failed:{failed}') + output = [''] + counts = {} + for platform, platform_data in res.items(): + output.append(platform_hdr.format(platform=platform)) + counts[platform] = {} + for distro, distro_data in platform_data.items(): + distro_failure_output = [] + output.append(distro_hdr.format(distro=distro)) + counts[platform][distro] = {'passed': 0, 'failed': 0} + for _, test_result in distro_data.items(): + if test_result['passed']: + counts[platform][distro]['passed'] += 1 + else: + counts[platform][distro]['failed'] += len( + test_result['failures']) + failure_output = format_test_failures(test_result) + if failure_output: + distro_failure_output.append(failure_output) + output.append( + distro_summary_fmt.format(**counts[platform][distro])) + if distro_failure_output: + output.extend(distro_failure_output) + return '\n'.join(output) + + def verify(args): """Verify test data. @@ -90,7 +135,7 @@ def verify(args): failed += len(fail_list) # dump results - LOG.debug('verify results: %s', res) + LOG.debug('\n---- Verify summarized results:\n%s', format_results(res)) if args.result: util.merge_results({'verify': res}, args.result) diff --git a/tests/data/netinfo/netdev-formatted-output b/tests/data/netinfo/netdev-formatted-output new file mode 100644 index 00000000..283ab4a4 --- /dev/null +++ b/tests/data/netinfo/netdev-formatted-output @@ -0,0 +1,10 @@ ++++++++++++++++++++++++++++++++++++++++Net device info++++++++++++++++++++++++++++++++++++++++ ++---------+------+------------------------------+---------------+--------+-------------------+ +| Device | Up | Address | Mask | Scope | Hw-Address | ++---------+------+------------------------------+---------------+--------+-------------------+ +| enp0s25 | True | 192.168.2.18 | 255.255.255.0 | . | 50:7b:9d:2c:af:91 | +| enp0s25 | True | fe80::7777:2222:1111:eeee/64 | . | global | 50:7b:9d:2c:af:91 | +| enp0s25 | True | fe80::8107:2b92:867e:f8a6/64 | . | link | 50:7b:9d:2c:af:91 | +| lo | True | 127.0.0.1 | 255.0.0.0 | . | . | +| lo | True | ::1/128 | . | host | . | ++---------+------+------------------------------+---------------+--------+-------------------+ diff --git a/tests/data/netinfo/netdev-formatted-output-down b/tests/data/netinfo/netdev-formatted-output-down new file mode 100644 index 00000000..038dfb4d --- /dev/null +++ b/tests/data/netinfo/netdev-formatted-output-down @@ -0,0 +1,8 @@ ++++++++++++++++++++++++++++Net device info++++++++++++++++++++++++++++ ++--------+-------+-----------+-----------+-------+-------------------+ +| Device | Up | Address | Mask | Scope | Hw-Address | ++--------+-------+-----------+-----------+-------+-------------------+ +| eth0 | False | . | . | . | 00:16:3e:de:51:a6 | +| lo | True | 127.0.0.1 | 255.0.0.0 | host | . | +| lo | True | ::1/128 | . | host | . | ++--------+-------+-----------+-----------+-------+-------------------+ diff --git a/tests/data/netinfo/new-ifconfig-output b/tests/data/netinfo/new-ifconfig-output new file mode 100644 index 00000000..83d4ad16 --- /dev/null +++ b/tests/data/netinfo/new-ifconfig-output @@ -0,0 +1,18 @@ +enp0s25: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 + inet 192.168.2.18 netmask 255.255.255.0 broadcast 192.168.2.255 + inet6 fe80::7777:2222:1111:eeee prefixlen 64 scopeid 0x30<global> + inet6 fe80::8107:2b92:867e:f8a6 prefixlen 64 scopeid 0x20<link> + ether 50:7b:9d:2c:af:91 txqueuelen 1000 (Ethernet) + RX packets 3017 bytes 10601563 (10.1 MiB) + RX errors 0 dropped 39 overruns 0 frame 0 + TX packets 2627 bytes 196976 (192.3 KiB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 + inet 127.0.0.1 netmask 255.0.0.0 + inet6 ::1 prefixlen 128 scopeid 0x10<host> + loop txqueuelen 1 (Local Loopback) + RX packets 0 bytes 0 (0.0 B) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 0 bytes 0 (0.0 B) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 diff --git a/tests/data/netinfo/new-ifconfig-output-down b/tests/data/netinfo/new-ifconfig-output-down new file mode 100644 index 00000000..5d12e352 --- /dev/null +++ b/tests/data/netinfo/new-ifconfig-output-down @@ -0,0 +1,15 @@ +eth0: flags=4098<BROADCAST,MULTICAST> mtu 1500 + ether 00:16:3e:de:51:a6 txqueuelen 1000 (Ethernet) + RX packets 126229 bytes 158139342 (158.1 MB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 59317 bytes 4839008 (4.8 MB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 + inet 127.0.0.1 netmask 255.0.0.0 + inet6 ::1 prefixlen 128 scopeid 0x10<host> + loop txqueuelen 1000 (Local Loopback) + RX packets 260 bytes 20092 (20.0 KB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 260 bytes 20092 (20.0 KB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 diff --git a/tests/data/netinfo/old-ifconfig-output b/tests/data/netinfo/old-ifconfig-output new file mode 100644 index 00000000..e01f763e --- /dev/null +++ b/tests/data/netinfo/old-ifconfig-output @@ -0,0 +1,18 @@ +enp0s25 Link encap:Ethernet HWaddr 50:7b:9d:2c:af:91 + inet addr:192.168.2.18 Bcast:192.168.2.255 Mask:255.255.255.0 + inet6 addr: fe80::7777:2222:1111:eeee/64 Scope:Global + inet6 addr: fe80::8107:2b92:867e:f8a6/64 Scope:Link + UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 + RX packets:8106427 errors:55 dropped:0 overruns:0 frame:37 + TX packets:9339739 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1000 + RX bytes:4953721719 (4.9 GB) TX bytes:7731890194 (7.7 GB) + Interrupt:20 Memory:e1200000-e1220000 + +lo Link encap:Local Loopback + inet addr:127.0.0.1 Mask:255.0.0.0 + inet6 addr: ::1/128 Scope:Host + UP LOOPBACK RUNNING MTU:65536 Metric:1 + RX packets:579230851 errors:0 dropped:0 overruns:0 frame:0 + TX packets:579230851 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1 diff --git a/tests/data/netinfo/route-formatted-output b/tests/data/netinfo/route-formatted-output new file mode 100644 index 00000000..9d2c5dd3 --- /dev/null +++ b/tests/data/netinfo/route-formatted-output @@ -0,0 +1,22 @@ ++++++++++++++++++++++++++++++Route IPv4 info+++++++++++++++++++++++++++++ ++-------+-------------+-------------+---------------+-----------+-------+ +| Route | Destination | Gateway | Genmask | Interface | Flags | ++-------+-------------+-------------+---------------+-----------+-------+ +| 0 | 0.0.0.0 | 192.168.2.1 | 0.0.0.0 | enp0s25 | UG | +| 1 | 0.0.0.0 | 192.168.2.1 | 0.0.0.0 | wlp3s0 | UG | +| 2 | 192.168.2.0 | 0.0.0.0 | 255.255.255.0 | enp0s25 | U | ++-------+-------------+-------------+---------------+-----------+-------+ ++++++++++++++++++++++++++++++++++++Route IPv6 info+++++++++++++++++++++++++++++++++++ ++-------+---------------------------+---------------------------+-----------+-------+ +| Route | Destination | Gateway | Interface | Flags | ++-------+---------------------------+---------------------------+-----------+-------+ +| 0 | 2a00:abcd:82ae:cd33::657 | :: | enp0s25 | Ue | +| 1 | 2a00:abcd:82ae:cd33::/64 | :: | enp0s25 | U | +| 2 | 2a00:abcd:82ae:cd33::/56 | fe80::32ee:54de:cd43:b4e1 | enp0s25 | UG | +| 3 | fd81:123f:654::657 | :: | enp0s25 | U | +| 4 | fd81:123f:654::/64 | :: | enp0s25 | U | +| 5 | fd81:123f:654::/48 | fe80::32ee:54de:cd43:b4e1 | enp0s25 | UG | +| 6 | fe80::abcd:ef12:bc34:da21 | :: | enp0s25 | U | +| 7 | fe80::/64 | :: | enp0s25 | U | +| 8 | ::/0 | fe80::32ee:54de:cd43:b4e1 | enp0s25 | UG | ++-------+---------------------------+---------------------------+-----------+-------+ diff --git a/tests/data/netinfo/sample-ipaddrshow-output b/tests/data/netinfo/sample-ipaddrshow-output new file mode 100644 index 00000000..b2fa2672 --- /dev/null +++ b/tests/data/netinfo/sample-ipaddrshow-output @@ -0,0 +1,13 @@ +1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever + inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever +2: enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP mode DEFAULT group default qlen 1000 + link/ether 50:7b:9d:2c:af:91 brd ff:ff:ff:ff:ff:ff + inet 192.168.2.18/24 brd 192.168.2.255 scope global dynamic enp0s25 + valid_lft 84174sec preferred_lft 84174sec + inet6 fe80::7777:2222:1111:eeee/64 scope global + valid_lft forever preferred_lft forever + inet6 fe80::8107:2b92:867e:f8a6/64 scope link + valid_lft forever preferred_lft forever + diff --git a/tests/data/netinfo/sample-ipaddrshow-output-down b/tests/data/netinfo/sample-ipaddrshow-output-down new file mode 100644 index 00000000..cb516d64 --- /dev/null +++ b/tests/data/netinfo/sample-ipaddrshow-output-down @@ -0,0 +1,8 @@ +1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +44: eth0@if45: <BROADCAST,MULTICAST> mtu 1500 qdisc noqueue state DOWN group default qlen 1000 + link/ether 00:16:3e:de:51:a6 brd ff:ff:ff:ff:ff:ff link-netnsid 0 diff --git a/tests/data/netinfo/sample-iproute-output-v4 b/tests/data/netinfo/sample-iproute-output-v4 new file mode 100644 index 00000000..904cb034 --- /dev/null +++ b/tests/data/netinfo/sample-iproute-output-v4 @@ -0,0 +1,3 @@ +default via 192.168.2.1 dev enp0s25 proto static metric 100 +default via 192.168.2.1 dev wlp3s0 proto static metric 150 +192.168.2.0/24 dev enp0s25 proto kernel scope link src 192.168.2.18 metric 100 diff --git a/tests/data/netinfo/sample-iproute-output-v6 b/tests/data/netinfo/sample-iproute-output-v6 new file mode 100644 index 00000000..12bb1c12 --- /dev/null +++ b/tests/data/netinfo/sample-iproute-output-v6 @@ -0,0 +1,11 @@ +2a00:abcd:82ae:cd33::657 dev enp0s25 proto kernel metric 256 expires 2334sec pref medium +2a00:abcd:82ae:cd33::/64 dev enp0s25 proto ra metric 100 pref medium +2a00:abcd:82ae:cd33::/56 via fe80::32ee:54de:cd43:b4e1 dev enp0s25 proto ra metric 100 pref medium +fd81:123f:654::657 dev enp0s25 proto kernel metric 256 pref medium +fd81:123f:654::/64 dev enp0s25 proto ra metric 100 pref medium +fd81:123f:654::/48 via fe80::32ee:54de:cd43:b4e1 dev enp0s25 proto ra metric 100 pref medium +fe80::abcd:ef12:bc34:da21 dev enp0s25 proto static metric 100 pref medium +fe80::/64 dev enp0s25 proto kernel metric 256 pref medium +default via fe80::32ee:54de:cd43:b4e1 dev enp0s25 proto static metric 100 pref medium +local ::1 dev lo table local proto none metric 0 pref medium +local 2600:1f16:b80:ad00:90a:c915:bca6:5ff2 dev lo table local proto none metric 0 pref medium diff --git a/tests/data/netinfo/sample-route-output-v4 b/tests/data/netinfo/sample-route-output-v4 new file mode 100644 index 00000000..ecc31d96 --- /dev/null +++ b/tests/data/netinfo/sample-route-output-v4 @@ -0,0 +1,5 @@ +Kernel IP routing table +Destination Gateway Genmask Flags Metric Ref Use Iface +0.0.0.0 192.168.2.1 0.0.0.0 UG 100 0 0 enp0s25 +0.0.0.0 192.168.2.1 0.0.0.0 UG 150 0 0 wlp3s0 +192.168.2.0 0.0.0.0 255.255.255.0 U 100 0 0 enp0s25 diff --git a/tests/data/netinfo/sample-route-output-v6 b/tests/data/netinfo/sample-route-output-v6 new file mode 100644 index 00000000..4712b73c --- /dev/null +++ b/tests/data/netinfo/sample-route-output-v6 @@ -0,0 +1,13 @@ +Kernel IPv6 routing table +Destination Next Hop Flag Met Re Use If +2a00:abcd:82ae:cd33::657/128 :: Ue 256 1 0 enp0s25 +2a00:abcd:82ae:cd33::/64 :: U 100 1 0 enp0s25 +2a00:abcd:82ae:cd33::/56 fe80::32ee:54de:cd43:b4e1 UG 100 1 0 enp0s25 +fd81:123f:654::657/128 :: U 256 1 0 enp0s25 +fd81:123f:654::/64 :: U 100 1 0 enp0s25 +fd81:123f:654::/48 fe80::32ee:54de:cd43:b4e1 UG 100 1 0 enp0s25 +fe80::abcd:ef12:bc34:da21/128 :: U 100 1 2 enp0s25 +fe80::/64 :: U 256 1 16880 enp0s25 +::/0 fe80::32ee:54de:cd43:b4e1 UG 100 1 0 enp0s25 +::/0 :: !n -1 1424956 lo +::1/128 :: Un 0 4 26289 lo diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 25878d7a..739bbebf 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -182,7 +182,7 @@ class TestCmdlineUrl(CiTestCase): self.assertEqual( ('url', 'http://example.com'), main.parse_cmdline_url(cmdline)) - @mock.patch('cloudinit.cmd.main.util.read_file_or_url') + @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url') def test_invalid_content(self, m_read): key = "cloud-config-url" url = 'http://example.com/foo' @@ -196,7 +196,7 @@ class TestCmdlineUrl(CiTestCase): self.assertIn(url, msg) self.assertFalse(os.path.exists(fpath)) - @mock.patch('cloudinit.cmd.main.util.read_file_or_url') + @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url') def test_valid_content(self, m_read): url = "http://example.com/foo" payload = b"#cloud-config\nmydata: foo\nbar: wark\n" @@ -210,18 +210,18 @@ class TestCmdlineUrl(CiTestCase): self.assertEqual(logging.INFO, lvl) self.assertIn(url, msg) - @mock.patch('cloudinit.cmd.main.util.read_file_or_url') + @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url') def test_no_key_found(self, m_read): cmdline = "ro mykey=http://example.com/foo root=foo" fpath = self.tmp_path("ccpath") - lvl, msg = main.attempt_cmdline_url( + lvl, _msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline) m_read.assert_not_called() self.assertFalse(os.path.exists(fpath)) self.assertEqual(logging.DEBUG, lvl) - @mock.patch('cloudinit.cmd.main.util.read_file_or_url') + @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url') def test_exception_warns(self, m_read): url = "http://example.com/foo" cmdline = "ro cloud-config-url=%s root=LABEL=bar" % url diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 9751ed95..abe820e1 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -2,27 +2,34 @@ """Tests of the built-in user data handlers.""" +import copy import os import shutil import tempfile +from textwrap import dedent -try: - from unittest import mock -except ImportError: - import mock -from cloudinit.tests import helpers as test_helpers +from cloudinit.tests.helpers import ( + FilesystemMockingTestCase, CiTestCase, mock, skipUnlessJinja) from cloudinit import handlers from cloudinit import helpers from cloudinit import util -from cloudinit.handlers import upstart_job +from cloudinit.handlers.cloud_config import CloudConfigPartHandler +from cloudinit.handlers.jinja_template import ( + JinjaTemplatePartHandler, convert_jinja_instance_data, + render_jinja_payload) +from cloudinit.handlers.shell_script import ShellScriptPartHandler +from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -class TestBuiltins(test_helpers.FilesystemMockingTestCase): +class TestUpstartJobPartHandler(FilesystemMockingTestCase): + + mpath = 'cloudinit.handlers.upstart_job.' + def test_upstart_frequency_no_out(self): c_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, c_root) @@ -32,14 +39,13 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): 'cloud_dir': c_root, 'upstart_dir': up_root, }) - freq = PER_ALWAYS - h = upstart_job.UpstartJobPartHandler(paths) + h = UpstartJobPartHandler(paths) # No files should be written out when # the frequency is ! per-instance h.handle_part('', handlers.CONTENT_START, None, None, None) h.handle_part('blah', 'text/upstart-job', - 'test.conf', 'blah', freq) + 'test.conf', 'blah', frequency=PER_ALWAYS) h.handle_part('', handlers.CONTENT_END, None, None, None) self.assertEqual(0, len(os.listdir(up_root))) @@ -48,7 +54,6 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): # files should be written out when frequency is ! per-instance new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) - freq = PER_INSTANCE self.patchOS(new_root) self.patchUtils(new_root) @@ -56,22 +61,297 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): 'upstart_dir': "/etc/upstart", }) - upstart_job.SUITABLE_UPSTART = True util.ensure_dir("/run") util.ensure_dir("/etc/upstart") - with mock.patch.object(util, 'subp') as mockobj: - h = upstart_job.UpstartJobPartHandler(paths) - h.handle_part('', handlers.CONTENT_START, - None, None, None) - h.handle_part('blah', 'text/upstart-job', - 'test.conf', 'blah', freq) - h.handle_part('', handlers.CONTENT_END, - None, None, None) + with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True): + with mock.patch.object(util, 'subp') as m_subp: + h = UpstartJobPartHandler(paths) + h.handle_part('', handlers.CONTENT_START, + None, None, None) + h.handle_part('blah', 'text/upstart-job', + 'test.conf', 'blah', frequency=PER_INSTANCE) + h.handle_part('', handlers.CONTENT_END, + None, None, None) - self.assertEqual(len(os.listdir('/etc/upstart')), 1) + self.assertEqual(len(os.listdir('/etc/upstart')), 1) - mockobj.assert_called_once_with( + m_subp.assert_called_once_with( ['initctl', 'reload-configuration'], capture=False) + +class TestJinjaTemplatePartHandler(CiTestCase): + + with_logs = True + + mpath = 'cloudinit.handlers.jinja_template.' + + def setUp(self): + super(TestJinjaTemplatePartHandler, self).setUp() + self.tmp = self.tmp_dir() + self.run_dir = os.path.join(self.tmp, 'run_dir') + util.ensure_dir(self.run_dir) + self.paths = helpers.Paths({ + 'cloud_dir': self.tmp, 'run_dir': self.run_dir}) + + def test_jinja_template_part_handler_defaults(self): + """On init, paths are saved and subhandler types are empty.""" + h = JinjaTemplatePartHandler(self.paths) + self.assertEqual(['## template: jinja'], h.prefixes) + self.assertEqual(3, h.handler_version) + self.assertEqual(self.paths, h.paths) + self.assertEqual({}, h.sub_handlers) + + def test_jinja_template_part_handler_looks_up_sub_handler_types(self): + """When sub_handlers are passed, init lists types of subhandlers.""" + script_handler = ShellScriptPartHandler(self.paths) + cloudconfig_handler = CloudConfigPartHandler(self.paths) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler, cloudconfig_handler]) + self.assertItemsEqual( + ['text/cloud-config', 'text/cloud-config-jsonp', + 'text/x-shellscript'], + h.sub_handlers) + + def test_jinja_template_part_handler_looks_up_subhandler_types(self): + """When sub_handlers are passed, init lists types of subhandlers.""" + script_handler = ShellScriptPartHandler(self.paths) + cloudconfig_handler = CloudConfigPartHandler(self.paths) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler, cloudconfig_handler]) + self.assertItemsEqual( + ['text/cloud-config', 'text/cloud-config-jsonp', + 'text/x-shellscript'], + h.sub_handlers) + + def test_jinja_template_handle_noop_on_content_signals(self): + """Perform no part handling when content type is CONTENT_SIGNALS.""" + script_handler = ShellScriptPartHandler(self.paths) + + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler]) + with mock.patch.object(script_handler, 'handle_part') as m_handle_part: + h.handle_part( + data='data', ctype=handlers.CONTENT_START, filename='part-1', + payload='## template: jinja\n#!/bin/bash\necho himom', + frequency='freq', headers='headers') + m_handle_part.assert_not_called() + + @skipUnlessJinja() + def test_jinja_template_handle_subhandler_v2_with_clean_payload(self): + """Call version 2 subhandler.handle_part with stripped payload.""" + script_handler = ShellScriptPartHandler(self.paths) + self.assertEqual(2, script_handler.handler_version) + + # Create required instance-data.json file + instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_data = {'topkey': 'echo himom'} + util.write_file(instance_json, util.json_dumps(instance_data)) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler]) + with mock.patch.object(script_handler, 'handle_part') as m_part: + # ctype with leading '!' not in handlers.CONTENT_SIGNALS + h.handle_part( + data='data', ctype="!" + handlers.CONTENT_START, + filename='part01', + payload='## template: jinja \t \n#!/bin/bash\n{{ topkey }}', + frequency='freq', headers='headers') + m_part.assert_called_once_with( + 'data', '!__begin__', 'part01', '#!/bin/bash\necho himom', 'freq') + + @skipUnlessJinja() + def test_jinja_template_handle_subhandler_v3_with_clean_payload(self): + """Call version 3 subhandler.handle_part with stripped payload.""" + cloudcfg_handler = CloudConfigPartHandler(self.paths) + self.assertEqual(3, cloudcfg_handler.handler_version) + + # Create required instance-data.json file + instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_data = {'topkey': {'sub': 'runcmd: [echo hi]'}} + util.write_file(instance_json, util.json_dumps(instance_data)) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[cloudcfg_handler]) + with mock.patch.object(cloudcfg_handler, 'handle_part') as m_part: + # ctype with leading '!' not in handlers.CONTENT_SIGNALS + h.handle_part( + data='data', ctype="!" + handlers.CONTENT_END, + filename='part01', + payload='## template: jinja\n#cloud-config\n{{ topkey.sub }}', + frequency='freq', headers='headers') + m_part.assert_called_once_with( + 'data', '!__end__', 'part01', '#cloud-config\nruncmd: [echo hi]', + 'freq', 'headers') + + def test_jinja_template_handle_errors_on_missing_instance_data_json(self): + """If instance-data is absent, raise an error from handle_part.""" + script_handler = ShellScriptPartHandler(self.paths) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler]) + with self.assertRaises(RuntimeError) as context_manager: + h.handle_part( + data='data', ctype="!" + handlers.CONTENT_START, + filename='part01', + payload='## template: jinja \n#!/bin/bash\necho himom', + frequency='freq', headers='headers') + script_file = os.path.join(script_handler.script_dir, 'part01') + self.assertEqual( + 'Cannot render jinja template vars. Instance data not yet present' + ' at {}/instance-data.json'.format( + self.run_dir), str(context_manager.exception)) + self.assertFalse( + os.path.exists(script_file), + 'Unexpected file created %s' % script_file) + + @skipUnlessJinja() + def test_jinja_template_handle_renders_jinja_content(self): + """When present, render jinja variables from instance-data.json.""" + script_handler = ShellScriptPartHandler(self.paths) + instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_data = {'topkey': {'subkey': 'echo himom'}} + util.write_file(instance_json, util.json_dumps(instance_data)) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler]) + h.handle_part( + data='data', ctype="!" + handlers.CONTENT_START, + filename='part01', + payload=( + '## template: jinja \n' + '#!/bin/bash\n' + '{{ topkey.subkey|default("nosubkey") }}'), + frequency='freq', headers='headers') + script_file = os.path.join(script_handler.script_dir, 'part01') + self.assertNotIn( + 'Instance data not yet present at {}/instance-data.json'.format( + self.run_dir), + self.logs.getvalue()) + self.assertEqual( + '#!/bin/bash\necho himom', util.load_file(script_file)) + + @skipUnlessJinja() + def test_jinja_template_handle_renders_jinja_content_missing_keys(self): + """When specified jinja variable is undefined, log a warning.""" + script_handler = ShellScriptPartHandler(self.paths) + instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_data = {'topkey': {'subkey': 'echo himom'}} + util.write_file(instance_json, util.json_dumps(instance_data)) + h = JinjaTemplatePartHandler( + self.paths, sub_handlers=[script_handler]) + h.handle_part( + data='data', ctype="!" + handlers.CONTENT_START, + filename='part01', + payload='## template: jinja \n#!/bin/bash\n{{ goodtry }}', + frequency='freq', headers='headers') + script_file = os.path.join(script_handler.script_dir, 'part01') + self.assertTrue( + os.path.exists(script_file), + 'Missing expected file %s' % script_file) + self.assertIn( + "WARNING: Could not render jinja template variables in file" + " 'part01': 'goodtry'\n", + self.logs.getvalue()) + + +class TestConvertJinjaInstanceData(CiTestCase): + + def test_convert_instance_data_hyphens_to_underscores(self): + """Replace hyphenated keys with underscores in instance-data.""" + data = {'hyphenated-key': 'hyphenated-val', + 'underscore_delim_key': 'underscore_delimited_val'} + expected_data = {'hyphenated_key': 'hyphenated-val', + 'underscore_delim_key': 'underscore_delimited_val'} + self.assertEqual( + expected_data, + convert_jinja_instance_data(data=data)) + + def test_convert_instance_data_promotes_versioned_keys_to_top_level(self): + """Any versioned keys are promoted as top-level keys + + This provides any cloud-init standardized keys up at a top-level to + allow ease of reference for users. Intsead of v1.availability_zone, + the name availability_zone can be used in templates. + """ + data = {'ds': {'dskey1': 1, 'dskey2': 2}, + 'v1': {'v1key1': 'v1.1'}, + 'v2': {'v2key1': 'v2.1'}} + expected_data = copy.deepcopy(data) + expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) + + converted_data = convert_jinja_instance_data(data=data) + self.assertItemsEqual( + ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) + self.assertEqual( + expected_data, + converted_data) + + def test_convert_instance_data_most_recent_version_of_promoted_keys(self): + """The most-recent versioned key value is promoted to top-level.""" + data = {'v1': {'key1': 'old v1 key1', 'key2': 'old v1 key2'}, + 'v2': {'key1': 'newer v2 key1', 'key3': 'newer v2 key3'}, + 'v3': {'key1': 'newest v3 key1'}} + expected_data = copy.deepcopy(data) + expected_data.update( + {'key1': 'newest v3 key1', 'key2': 'old v1 key2', + 'key3': 'newer v2 key3'}) + + converted_data = convert_jinja_instance_data(data=data) + self.assertEqual( + expected_data, + converted_data) + + def test_convert_instance_data_decodes_decode_paths(self): + """Any decode_paths provided are decoded by convert_instance_data.""" + data = {'key1': {'subkey1': 'aGkgbW9t'}, 'key2': 'aGkgZGFk'} + expected_data = copy.deepcopy(data) + expected_data['key1']['subkey1'] = 'hi mom' + + converted_data = convert_jinja_instance_data( + data=data, decode_paths=('key1/subkey1',)) + self.assertEqual( + expected_data, + converted_data) + + +class TestRenderJinjaPayload(CiTestCase): + + with_logs = True + + @skipUnlessJinja() + def test_render_jinja_payload_logs_jinja_vars_on_debug(self): + """When debug is True, log jinja varables available.""" + payload = ( + '## template: jinja\n#!/bin/sh\necho hi from {{ v1.hostname }}') + instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'} + expected_log = dedent("""\ + DEBUG: Converted jinja variables + { + "hostname": "foo", + "instance_id": "iid", + "v1": { + "hostname": "foo" + } + } + """) + self.assertEqual( + render_jinja_payload( + payload=payload, payload_fn='myfile', + instance_data=instance_data, debug=True), + '#!/bin/sh\necho hi from foo') + self.assertEqual(expected_log, self.logs.getvalue()) + + @skipUnlessJinja() + def test_render_jinja_payload_replaces_missing_variables_and_warns(self): + """Warn on missing jinja variables and replace the absent variable.""" + payload = ( + '## template: jinja\n#!/bin/sh\necho hi from {{ NOTHERE }}') + instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'} + self.assertEqual( + render_jinja_payload( + payload=payload, payload_fn='myfile', + instance_data=instance_data), + '#!/bin/sh\necho hi from CI_MISSING_JINJA_VAR/NOTHERE') + expected_log = ( + 'WARNING: Could not render jinja template variables in file' + " 'myfile': 'NOTHERE'") + self.assertIn(expected_log, self.logs.getvalue()) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 0c0f427a..199d69b0 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -208,8 +208,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): for subcommand in expected_subcommands: self.assertIn(subcommand, error) - @mock.patch('cloudinit.config.schema.handle_schema_args') - def test_wb_devel_schema_subcommand_parser(self, m_schema): + def test_wb_devel_schema_subcommand_parser(self): """The subcommand cloud-init schema calls the correct subparser.""" exit_code = self._call_main(['cloud-init', 'devel', 'schema']) self.assertEqual(1, exit_code) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 275b16d2..3efe7adf 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -524,7 +524,17 @@ c: 4 self.assertEqual(cfg.get('password'), 'gocubs') self.assertEqual(cfg.get('locale'), 'chicago') - @httpretty.activate + +class TestConsumeUserDataHttp(TestConsumeUserData, helpers.HttprettyTestCase): + + def setUp(self): + TestConsumeUserData.setUp(self) + helpers.HttprettyTestCase.setUp(self) + + def tearDown(self): + TestConsumeUserData.tearDown(self) + helpers.HttprettyTestCase.tearDown(self) + @mock.patch('cloudinit.url_helper.time.sleep') def test_include(self, mock_sleep): """Test #include.""" @@ -543,7 +553,6 @@ c: 4 cc = util.load_yaml(cc_contents) self.assertTrue(cc.get('included')) - @httpretty.activate @mock.patch('cloudinit.url_helper.time.sleep') def test_include_bad_url(self, mock_sleep): """Test #include with a bad URL.""" @@ -597,8 +606,10 @@ class TestUDProcess(helpers.ResourceUsingTestCase): class TestConvertString(helpers.TestCase): + def test_handles_binary_non_utf8_decodable(self): - blob = b'\x32\x99' + """Printable unicode (not utf8-decodable) is safely converted.""" + blob = b'#!/bin/bash\necho \xc3\x84\n' msg = ud.convert_string(blob) self.assertEqual(blob, msg.get_payload(decode=True)) @@ -612,6 +623,13 @@ class TestConvertString(helpers.TestCase): msg = ud.convert_string(text) self.assertEqual(text, msg.get_payload(decode=False)) + def test_handle_mime_parts(self): + """Mime parts are properly returned as a mime message.""" + message = MIMEBase("text", "plain") + message.set_payload("Just text") + msg = ud.convert_string(str(message)) + self.assertEqual("Just text", msg.get_payload(decode=False)) + class TestFetchBaseConfig(helpers.TestCase): def test_only_builtin_gets_builtin(self): diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 4fa9616b..1e77842f 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -130,7 +130,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase): self.ds.get_hostname()) @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") - @httpretty.activate def test_with_mock_server(self, m_is_aliyun): m_is_aliyun.return_value = True self.regist_default_server() @@ -143,7 +142,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase): self._test_host_name() @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") - @httpretty.activate def test_returns_false_when_not_on_aliyun(self, m_is_aliyun): """If is_aliyun returns false, then get_data should return False.""" m_is_aliyun.return_value = False diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 3253f3ad..ff35904e 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -262,64 +262,56 @@ class TestUserDataRhevm(CiTestCase): ''' Test to exercise method: DataSourceAltCloud.user_data_rhevm() ''' - cmd_pass = ['true'] - cmd_fail = ['false'] - cmd_not_found = ['bogus bad command'] - def setUp(self): '''Set up.''' self.paths = helpers.Paths({'cloud_dir': '/tmp'}) - self.mount_dir = tempfile.mkdtemp() + self.mount_dir = self.tmp_dir() _write_user_data_files(self.mount_dir, 'test user data') - - def tearDown(self): - # Reset - - _remove_user_data_files(self.mount_dir) - - # Attempt to remove the temp dir ignoring errors - try: - shutil.rmtree(self.mount_dir) - except OSError: - pass - - dsac.CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info' - dsac.CMD_PROBE_FLOPPY = ['modprobe', 'floppy'] - dsac.CMD_UDEVADM_SETTLE = ['udevadm', 'settle', - '--quiet', '--timeout=5'] + self.add_patch( + 'cloudinit.sources.DataSourceAltCloud.modprobe_floppy', + 'm_modprobe_floppy', return_value=None) + self.add_patch( + 'cloudinit.sources.DataSourceAltCloud.util.udevadm_settle', + 'm_udevadm_settle', return_value=('', '')) + self.add_patch( + 'cloudinit.sources.DataSourceAltCloud.util.mount_cb', + 'm_mount_cb') def test_mount_cb_fails(self): '''Test user_data_rhevm() where mount_cb fails.''' - dsac.CMD_PROBE_FLOPPY = self.cmd_pass + self.m_mount_cb.side_effect = util.MountFailedError("Failed Mount") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) def test_modprobe_fails(self): '''Test user_data_rhevm() where modprobe fails.''' - dsac.CMD_PROBE_FLOPPY = self.cmd_fail + self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + "Failed modprobe") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_modprobe_cmd(self): '''Test user_data_rhevm() with no modprobe command.''' - dsac.CMD_PROBE_FLOPPY = self.cmd_not_found + self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + "No such file or dir") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) def test_udevadm_fails(self): '''Test user_data_rhevm() where udevadm fails.''' - dsac.CMD_UDEVADM_SETTLE = self.cmd_fail + self.m_udevadm_settle.side_effect = util.ProcessExecutionError( + "Failed settle.") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_udevadm_cmd(self): '''Test user_data_rhevm() with no udevadm command.''' - dsac.CMD_UDEVADM_SETTLE = self.cmd_not_found + self.m_udevadm_settle.side_effect = OSError("No such file or dir") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3e8b7913..4e428b71 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,15 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. +from cloudinit import distros from cloudinit import helpers -from cloudinit.util import b64e, decode_binary, load_file, write_file -from cloudinit.sources import DataSourceAzure as dsaz -from cloudinit.util import find_freebsd_part -from cloudinit.util import get_path_dev_freebsd +from cloudinit import url_helper +from cloudinit.sources import ( + UNSET, DataSourceAzure as dsaz, InvalidMetaDataException) +from cloudinit.util import (b64e, decode_binary, load_file, write_file, + find_freebsd_part, get_path_dev_freebsd, + MountFailedError) from cloudinit.version import version_string as vs -from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) +from cloudinit.tests.helpers import ( + HttprettyTestCase, CiTestCase, populate_dir, mock, wrap_and_call, + ExitStack, PY26, SkipTest) import crypt +import httpretty +import json import os import stat import xml.etree.ElementTree as ET @@ -77,6 +83,106 @@ def construct_valid_ovf_env(data=None, pubkeys=None, return content +NETWORK_METADATA = { + "network": { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "ipAddress": [] + }, + "ipv4": { + "subnet": [ + { + "prefix": "24", + "address": "10.0.0.0" + } + ], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81" + } + ] + } + } + ] + } +} + + +class TestGetMetadataFromIMDS(HttprettyTestCase): + + with_logs = True + + def setUp(self): + super(TestGetMetadataFromIMDS, self).setUp() + self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2017-12-01" + + @mock.patch('cloudinit.sources.DataSourceAzure.readurl') + @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4') + @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up') + def test_get_metadata_does_not_dhcp_if_network_is_up( + self, m_net_is_up, m_dhcp, m_readurl): + """Do not perform DHCP setup when nic is already up.""" + m_net_is_up.return_value = True + m_readurl.return_value = url_helper.StringResponse( + json.dumps(NETWORK_METADATA).encode('utf-8')) + self.assertEqual( + NETWORK_METADATA, + dsaz.get_metadata_from_imds('eth9', retries=3)) + + m_net_is_up.assert_called_with('eth9') + m_dhcp.assert_not_called() + self.assertIn( + "Crawl of Azure Instance Metadata Service (IMDS) took", # log_time + self.logs.getvalue()) + + @mock.patch('cloudinit.sources.DataSourceAzure.readurl') + @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4') + @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up') + def test_get_metadata_performs_dhcp_when_network_is_down( + self, m_net_is_up, m_dhcp, m_readurl): + """Perform DHCP setup when nic is not up.""" + m_net_is_up.return_value = False + m_readurl.return_value = url_helper.StringResponse( + json.dumps(NETWORK_METADATA).encode('utf-8')) + + self.assertEqual( + NETWORK_METADATA, + dsaz.get_metadata_from_imds('eth9', retries=2)) + + m_net_is_up.assert_called_with('eth9') + m_dhcp.assert_called_with('eth9') + self.assertIn( + "Crawl of Azure Instance Metadata Service (IMDS) took", # log_time + self.logs.getvalue()) + + m_readurl.assert_called_with( + self.network_md_url, exception_cb=mock.ANY, + headers={'Metadata': 'true'}, retries=2, timeout=1) + + @mock.patch('cloudinit.url_helper.time.sleep') + @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up') + def test_get_metadata_from_imds_empty_when_no_imds_present( + self, m_net_is_up, m_sleep): + """Return empty dict when IMDS network metadata is absent.""" + httpretty.register_uri( + httpretty.GET, + dsaz.IMDS_URL + 'instance?api-version=2017-12-01', + body={}, status=404) + + m_net_is_up.return_value = True # skips dhcp + + self.assertEqual({}, dsaz.get_metadata_from_imds('eth9', retries=2)) + + m_net_is_up.assert_called_with('eth9') + self.assertEqual([mock.call(1), mock.call(1)], m_sleep.call_args_list) + self.assertIn( + "Crawl of Azure Instance Metadata Service (IMDS) took", # log_time + self.logs.getvalue()) + + class TestAzureDataSource(CiTestCase): with_logs = True @@ -95,6 +201,19 @@ class TestAzureDataSource(CiTestCase): self.patches = ExitStack() self.addCleanup(self.patches.close) + self.patches.enter_context(mock.patch.object( + dsaz, '_get_random_seed', return_value='wild')) + self.m_get_metadata_from_imds = self.patches.enter_context( + mock.patch.object( + dsaz, 'get_metadata_from_imds', + mock.MagicMock(return_value=NETWORK_METADATA))) + self.m_fallback_nic = self.patches.enter_context( + mock.patch('cloudinit.sources.net.find_fallback_nic', + return_value='eth9')) + self.m_remove_ubuntu_network_scripts = self.patches.enter_context( + mock.patch.object( + dsaz, 'maybe_remove_ubuntu_network_config_scripts', + mock.MagicMock())) super(TestAzureDataSource, self).setUp() def apply_patches(self, patches): @@ -135,7 +254,7 @@ scbus-1 on xpt0 bus 0 ]) return dsaz - def _get_ds(self, data, agent_command=None): + def _get_ds(self, data, agent_command=None, distro=None): def dsdevs(): return data.get('dsdevs', []) @@ -184,8 +303,11 @@ scbus-1 on xpt0 bus 0 side_effect=_wait_for_files)), ]) + if distro is not None: + distro_cls = distros.fetch(distro) + distro = distro_cls(distro, data.get('sys_cfg', {}), self.paths) dsrc = dsaz.DataSourceAzure( - data.get('sys_cfg', {}), distro=None, paths=self.paths) + data.get('sys_cfg', {}), distro=distro, paths=self.paths) if agent_command is not None: dsrc.ds_cfg['agent_command'] = agent_command @@ -214,7 +336,7 @@ scbus-1 on xpt0 bus 0 self.assertIn(tag, x) def tags_equal(x, y): - for x_tag, x_val in x.items(): + for x_val in x.values(): y_val = y.get(x_val.tag) self.assertEqual(x_val.text, y_val.text) @@ -258,29 +380,20 @@ fdescfs /dev/fd fdescfs rw 0 0 res = get_path_dev_freebsd('/etc', mnt_list) self.assertIsNotNone(res) - @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data') - def test_non_azure_dmi_chassis_asset_tag(self, m_read_dmi_data): - """Report non-azure when DMI's chassis asset tag doesn't match. - - Return False when the asset tag doesn't match Azure's static - AZURE_CHASSIS_ASSET_TAG. - """ + @mock.patch('cloudinit.sources.DataSourceAzure._is_platform_viable') + def test_call_is_platform_viable_seed(self, m_is_platform_viable): + """Check seed_dir using _is_platform_viable and return False.""" # Return a non-matching asset tag value - nonazure_tag = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X' - m_read_dmi_data.return_value = nonazure_tag + m_is_platform_viable.return_value = False dsrc = dsaz.DataSourceAzure( {}, distro=None, paths=self.paths) self.assertFalse(dsrc.get_data()) - self.assertEqual( - "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( - nonazure_tag), - self.logs.getvalue()) + m_is_platform_viable.assert_called_with(dsrc.seed_dir) def test_basic_seed_dir(self): odata = {'HostName': "myhost", 'UserName': "myuser"} data = {'ovfcontent': construct_valid_ovf_env(data=odata), 'sys_cfg': {}} - dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) @@ -289,6 +402,82 @@ fdescfs /dev/fd fdescfs rw 0 0 self.assertTrue(os.path.isfile( os.path.join(self.waagent_d, 'ovf-env.xml'))) + def test_get_data_non_ubuntu_will_not_remove_network_scripts(self): + """get_data on non-Ubuntu will not remove ubuntu net scripts.""" + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + dsrc = self._get_ds(data, distro='debian') + dsrc.get_data() + self.m_remove_ubuntu_network_scripts.assert_not_called() + + def test_get_data_on_ubuntu_will_remove_network_scripts(self): + """get_data will remove ubuntu net scripts on Ubuntu distro.""" + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + dsrc = self._get_ds(data, distro='ubuntu') + dsrc.get_data() + self.m_remove_ubuntu_network_scripts.assert_called_once_with() + + def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): + """Return all structured metadata and cache no class attributes.""" + yaml_cfg = "{agent_command: my_command}\n" + odata = {'HostName': "myhost", 'UserName': "myuser", + 'UserData': {'text': 'FOOBAR', 'encoding': 'plain'}, + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + dsrc = self._get_ds(data) + expected_cfg = { + 'PreprovisionedVm': False, + 'datasource': {'Azure': {'agent_command': 'my_command'}}, + 'system_info': {'default_user': {'name': u'myuser'}}} + expected_metadata = { + 'azure_data': { + 'configurationsettype': 'LinuxProvisioningConfiguration'}, + 'imds': {'network': {'interface': [{ + 'ipv4': {'ipAddress': [ + {'privateIpAddress': '10.0.0.4', + 'publicIpAddress': '104.46.124.81'}], + 'subnet': [{'address': '10.0.0.0', 'prefix': '24'}]}, + 'ipv6': {'ipAddress': []}, + 'macAddress': '000D3A047598'}]}}, + 'instance-id': 'test-instance-id', + 'local-hostname': u'myhost', + 'random_seed': 'wild'} + + crawled_metadata = dsrc.crawl_metadata() + + self.assertItemsEqual( + crawled_metadata.keys(), + ['cfg', 'files', 'metadata', 'userdata_raw']) + self.assertEqual(crawled_metadata['cfg'], expected_cfg) + self.assertEqual( + list(crawled_metadata['files'].keys()), ['ovf-env.xml']) + self.assertIn( + b'<HostName>myhost</HostName>', + crawled_metadata['files']['ovf-env.xml']) + self.assertEqual(crawled_metadata['metadata'], expected_metadata) + self.assertEqual(crawled_metadata['userdata_raw'], 'FOOBAR') + self.assertEqual(dsrc.userdata_raw, None) + self.assertEqual(dsrc.metadata, {}) + self.assertEqual(dsrc._metadata_imds, UNSET) + self.assertFalse(os.path.isfile( + os.path.join(self.waagent_d, 'ovf-env.xml'))) + + def test_crawl_metadata_raises_invalid_metadata_on_error(self): + """crawl_metadata raises an exception on invalid ovf-env.xml.""" + data = {'ovfcontent': "BOGUS", 'sys_cfg': {}} + dsrc = self._get_ds(data) + error_msg = ('BrokenAzureDataSource: Invalid ovf-env.xml:' + ' syntax error: line 1, column 0') + with self.assertRaises(InvalidMetaDataException) as cm: + dsrc.crawl_metadata() + self.assertEqual(str(cm.exception), error_msg) + def test_waagent_d_has_0700_perms(self): # we expect /var/lib/waagent to be created 0700 dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) @@ -312,6 +501,20 @@ fdescfs /dev/fd fdescfs rw 0 0 self.assertTrue(ret) self.assertEqual(data['agent_invoked'], cfg['agent_command']) + def test_network_config_set_from_imds(self): + """Datasource.network_config returns IMDS network data.""" + odata = {} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + expected_network_config = { + 'ethernets': { + 'eth0': {'set-name': 'eth0', + 'match': {'macaddress': '00:0d:3a:04:75:98'}, + 'dhcp4': True}}, + 'version': 2} + dsrc = self._get_ds(data) + dsrc.get_data() + self.assertEqual(expected_network_config, dsrc.network_config) + def test_user_cfg_set_agent_command(self): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} @@ -335,6 +538,18 @@ fdescfs /dev/fd fdescfs rw 0 0 self.assertTrue(ret) self.assertEqual(data['agent_invoked'], '_COMMAND') + def test_sys_cfg_set_never_destroy_ntfs(self): + sys_cfg = {'datasource': {'Azure': { + 'never_destroy_ntfs': 'user-supplied-value'}}} + data = {'ovfcontent': construct_valid_ovf_env(data={}), + 'sys_cfg': sys_cfg} + + dsrc = self._get_ds(data) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) + self.assertEqual(dsrc.ds_cfg.get(dsaz.DS_CFG_KEY_PRESERVE_NTFS), + 'user-supplied-value') + def test_username_used(self): odata = {'HostName': "myhost", 'UserName': "myuser"} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} @@ -565,12 +780,34 @@ fdescfs /dev/fd fdescfs rw 0 0 self.assertEqual( [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list) + @mock.patch('cloudinit.net.generate_fallback_config') + def test_imds_network_config(self, mock_fallback): + """Network config is generated from IMDS network data when present.""" + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + + expected_cfg = { + 'ethernets': { + 'eth0': {'dhcp4': True, + 'match': {'macaddress': '00:0d:3a:04:75:98'}, + 'set-name': 'eth0'}}, + 'version': 2} + + self.assertEqual(expected_cfg, dsrc.network_config) + mock_fallback.assert_not_called() + @mock.patch('cloudinit.net.get_interface_mac') @mock.patch('cloudinit.net.get_devicelist') @mock.patch('cloudinit.net.device_driver') @mock.patch('cloudinit.net.generate_fallback_config') - def test_network_config(self, mock_fallback, mock_dd, - mock_devlist, mock_get_mac): + def test_fallback_network_config(self, mock_fallback, mock_dd, + mock_devlist, mock_get_mac): + """On absent IMDS network data, generate network fallback config.""" odata = {'HostName': "myhost", 'UserName': "myuser"} data = {'ovfcontent': construct_valid_ovf_env(data=odata), 'sys_cfg': {}} @@ -591,6 +828,8 @@ fdescfs /dev/fd fdescfs rw 0 0 mock_get_mac.return_value = '00:11:22:33:44:55' dsrc = self._get_ds(data) + # Represent empty response from network imds + self.m_get_metadata_from_imds.return_value = {} ret = dsrc.get_data() self.assertTrue(ret) @@ -603,8 +842,9 @@ fdescfs /dev/fd fdescfs rw 0 0 @mock.patch('cloudinit.net.get_devicelist') @mock.patch('cloudinit.net.device_driver') @mock.patch('cloudinit.net.generate_fallback_config') - def test_network_config_blacklist(self, mock_fallback, mock_dd, - mock_devlist, mock_get_mac): + def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd, + mock_devlist, mock_get_mac): + """On absent network metadata, blacklist mlx from fallback config.""" odata = {'HostName': "myhost", 'UserName': "myuser"} data = {'ovfcontent': construct_valid_ovf_env(data=odata), 'sys_cfg': {}} @@ -635,6 +875,8 @@ fdescfs /dev/fd fdescfs rw 0 0 mock_get_mac.return_value = '00:11:22:33:44:55' dsrc = self._get_ds(data) + # Represent empty response from network imds + self.m_get_metadata_from_imds.return_value = {} ret = dsrc.get_data() self.assertTrue(ret) @@ -675,7 +917,12 @@ class TestAzureBounce(CiTestCase): mock.patch.object(dsaz, 'get_metadata_from_fabric', mock.MagicMock(return_value={}))) self.patches.enter_context( + mock.patch.object(dsaz, 'get_metadata_from_imds', + mock.MagicMock(return_value={}))) + self.patches.enter_context( mock.patch.object(dsaz.util, 'which', lambda x: True)) + self.patches.enter_context(mock.patch.object( + dsaz, '_get_random_seed', return_value='wild')) def _dmi_mocks(key): if key == 'system-uuid': @@ -703,9 +950,12 @@ class TestAzureBounce(CiTestCase): mock.patch.object(dsaz, 'set_hostname')) self.subp = self.patches.enter_context( mock.patch('cloudinit.sources.DataSourceAzure.util.subp')) + self.find_fallback_nic = self.patches.enter_context( + mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9')) def tearDown(self): self.patches.close() + super(TestAzureBounce, self).tearDown() def _get_ds(self, ovfcontent=None, agent_command=None): if ovfcontent is not None: @@ -911,7 +1161,7 @@ class TestLoadAzureDsDir(CiTestCase): str(context_manager.exception)) -class TestReadAzureOvf(TestCase): +class TestReadAzureOvf(CiTestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "<foo>" + construct_valid_ovf_env(data={}) @@ -957,7 +1207,9 @@ class TestCanDevBeReformatted(CiTestCase): # return sorted by partition number return sorted(ret, key=lambda d: d[0]) - def mount_cb(device, callback): + def mount_cb(device, callback, mtype, update_env_for_mount): + self.assertEqual('ntfs', mtype) + self.assertEqual('C', update_env_for_mount.get('LANG')) p = self.tmp_dir() for f in bypath.get(device).get('files', []): write_file(os.path.join(p, f), content=f) @@ -988,14 +1240,16 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda2': {'num': 2}, '/dev/sda3': {'num': 3}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertFalse(value) self.assertIn("3 or more", msg.lower()) def test_no_partitions_is_false(self): """A disk with no partitions can not be formatted.""" self.patchup({'/dev/sda': {}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertFalse(value) self.assertIn("not partitioned", msg.lower()) @@ -1007,7 +1261,8 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1}, '/dev/sda2': {'num': 2, 'fs': 'ext4', 'files': []}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertFalse(value) self.assertIn("not ntfs", msg.lower()) @@ -1020,7 +1275,8 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda2': {'num': 2, 'fs': 'ntfs', 'files': ['secret.txt']}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertFalse(value) self.assertIn("files on it", msg.lower()) @@ -1032,7 +1288,8 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1}, '/dev/sda2': {'num': 2, 'fs': 'ntfs', 'files': []}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertTrue(value) self.assertIn("safe for", msg.lower()) @@ -1043,7 +1300,8 @@ class TestCanDevBeReformatted(CiTestCase): 'partitions': { '/dev/sda1': {'num': 1, 'fs': 'zfs'}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertFalse(value) self.assertIn("not ntfs", msg.lower()) @@ -1055,9 +1313,14 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': ['file1.txt', 'file2.exe']}, }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertFalse(value) - self.assertIn("files on it", msg.lower()) + with mock.patch.object(dsaz.LOG, 'warning') as warning: + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) + wmsg = warning.call_args[0][0] + self.assertIn("looks like you're using NTFS on the ephemeral disk", + wmsg) + self.assertFalse(value) + self.assertIn("files on it", msg.lower()) def test_one_partition_ntfs_empty_is_true(self): """1 mountable ntfs partition and no files can be formatted.""" @@ -1066,7 +1329,8 @@ class TestCanDevBeReformatted(CiTestCase): 'partitions': { '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []} }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertTrue(value) self.assertIn("safe for", msg.lower()) @@ -1078,7 +1342,8 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': ['dataloss_warning_readme.txt']} }}}) - value, msg = dsaz.can_dev_be_reformatted("/dev/sda") + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=False) self.assertTrue(value) self.assertIn("safe for", msg.lower()) @@ -1093,7 +1358,8 @@ class TestCanDevBeReformatted(CiTestCase): 'num': 1, 'fs': 'ntfs', 'files': [self.warning_file], 'realpath': '/dev/sdb1'} }}}) - value, msg = dsaz.can_dev_be_reformatted(epath) + value, msg = dsaz.can_dev_be_reformatted(epath, + preserve_ntfs=False) self.assertTrue(value) self.assertIn("safe for", msg.lower()) @@ -1112,10 +1378,68 @@ class TestCanDevBeReformatted(CiTestCase): epath + '-part3': {'num': 3, 'fs': 'ext', 'realpath': '/dev/sdb3'} }}}) - value, msg = dsaz.can_dev_be_reformatted(epath) + value, msg = dsaz.can_dev_be_reformatted(epath, + preserve_ntfs=False) self.assertFalse(value) self.assertIn("3 or more", msg.lower()) + def test_ntfs_mount_errors_true(self): + """can_dev_be_reformatted does not fail if NTFS is unknown fstype.""" + self.patchup({ + '/dev/sda': { + 'partitions': { + '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []} + }}}) + + err = ("Unexpected error while running command.\n", + "Command: ['mount', '-o', 'ro,sync', '-t', 'auto', ", + "'/dev/sda1', '/fake-tmp/dir']\n" + "Exit code: 32\n" + "Reason: -\n" + "Stdout: -\n" + "Stderr: mount: unknown filesystem type 'ntfs'") + self.m_mount_cb.side_effect = MountFailedError( + 'Failed mounting %s to %s due to: %s' % + ('/dev/sda', '/fake-tmp/dir', err)) + + value, msg = dsaz.can_dev_be_reformatted('/dev/sda', + preserve_ntfs=False) + self.assertTrue(value) + self.assertIn('cannot mount NTFS, assuming', msg) + + def test_never_destroy_ntfs_config_false(self): + """Normally formattable situation with never_destroy_ntfs set.""" + self.patchup({ + '/dev/sda': { + 'partitions': { + '/dev/sda1': {'num': 1, 'fs': 'ntfs', + 'files': ['dataloss_warning_readme.txt']} + }}}) + value, msg = dsaz.can_dev_be_reformatted("/dev/sda", + preserve_ntfs=True) + self.assertFalse(value) + self.assertIn("config says to never destroy NTFS " + "(datasource.Azure.never_destroy_ntfs)", msg) + + +class TestClearCachedData(CiTestCase): + + def test_clear_cached_attrs_clears_imds(self): + """All class attributes are reset to defaults, including imds data.""" + tmp = self.tmp_dir() + paths = helpers.Paths( + {'cloud_dir': tmp, 'run_dir': tmp}) + dsrc = dsaz.DataSourceAzure({}, distro=None, paths=paths) + clean_values = [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds] + dsrc.metadata = 'md' + dsrc.userdata = 'ud' + dsrc._metadata_imds = 'imds' + dsrc._dirty_cache = True + dsrc.clear_cached_attrs() + self.assertEqual( + [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds], + clean_values) + class TestAzureNetExists(CiTestCase): @@ -1125,19 +1449,9 @@ class TestAzureNetExists(CiTestCase): self.assertTrue(hasattr(dsaz, "DataSourceAzureNet")) -@mock.patch('cloudinit.sources.DataSourceAzure.util.subp') -@mock.patch.object(dsaz, 'get_hostname') -@mock.patch.object(dsaz, 'set_hostname') -class TestAzureDataSourcePreprovisioning(CiTestCase): - - def setUp(self): - super(TestAzureDataSourcePreprovisioning, self).setUp() - tmp = self.tmp_dir() - self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) - self.paths = helpers.Paths({'cloud_dir': tmp}) - dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d +class TestPreprovisioningReadAzureOvfFlag(CiTestCase): - def test_read_azure_ovf_with_true_flag(self, *args): + def test_read_azure_ovf_with_true_flag(self): """The read_azure_ovf method should set the PreprovisionedVM cfg flag if the proper setting is present.""" content = construct_valid_ovf_env( @@ -1146,7 +1460,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): cfg = ret[2] self.assertTrue(cfg['PreprovisionedVm']) - def test_read_azure_ovf_with_false_flag(self, *args): + def test_read_azure_ovf_with_false_flag(self): """The read_azure_ovf method should set the PreprovisionedVM cfg flag to false if the proper setting is false.""" content = construct_valid_ovf_env( @@ -1155,7 +1469,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): cfg = ret[2] self.assertFalse(cfg['PreprovisionedVm']) - def test_read_azure_ovf_without_flag(self, *args): + def test_read_azure_ovf_without_flag(self): """The read_azure_ovf method should not set the PreprovisionedVM cfg flag.""" content = construct_valid_ovf_env() @@ -1163,12 +1477,121 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): cfg = ret[2] self.assertFalse(cfg['PreprovisionedVm']) - @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') - @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') - @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') - @mock.patch('requests.Session.request') + +@mock.patch('os.path.isfile') +class TestPreprovisioningShouldReprovision(CiTestCase): + + def setUp(self): + super(TestPreprovisioningShouldReprovision, self).setUp() + tmp = self.tmp_dir() + self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) + self.paths = helpers.Paths({'cloud_dir': tmp}) + dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d + + @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') + def test__should_reprovision_with_true_cfg(self, isfile, write_f): + """The _should_reprovision method should return true with config + flag present.""" + isfile.return_value = False + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + self.assertTrue(dsa._should_reprovision( + (None, None, {'PreprovisionedVm': True}, None))) + + def test__should_reprovision_with_file_existing(self, isfile): + """The _should_reprovision method should return True if the sentinal + exists.""" + isfile.return_value = True + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + self.assertTrue(dsa._should_reprovision( + (None, None, {'preprovisionedvm': False}, None))) + + def test__should_reprovision_returns_false(self, isfile): + """The _should_reprovision method should return False + if config and sentinal are not present.""" + isfile.return_value = False + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + self.assertFalse(dsa._should_reprovision((None, None, {}, None))) + + @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') + def test_reprovision_calls__poll_imds(self, _poll_imds, isfile): + """_reprovision will poll IMDS.""" + isfile.return_value = False + hostname = "myhost" + username = "myuser" + odata = {'HostName': hostname, 'UserName': username} + _poll_imds.return_value = construct_valid_ovf_env(data=odata) + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa._reprovision() + _poll_imds.assert_called_with() + + +@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') +@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') +@mock.patch('requests.Session.request') +@mock.patch( + 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') +class TestPreprovisioningPollIMDS(CiTestCase): + + def setUp(self): + super(TestPreprovisioningPollIMDS, self).setUp() + self.tmp = self.tmp_dir() + self.waagent_d = self.tmp_path('/var/lib/waagent', self.tmp) + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d + + @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') + def test_poll_imds_calls_report_ready(self, write_f, report_ready_func, + fake_resp, m_dhcp, m_net): + """The poll_imds will call report_ready after creating marker file.""" + report_marker = self.tmp_path('report_marker', self.tmp) + lease = { + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'} + m_dhcp.return_value = [lease] + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + mock_path = ( + 'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE') + with mock.patch(mock_path, report_marker): + dsa._poll_imds() + self.assertEqual(report_ready_func.call_count, 1) + report_ready_func.assert_called_with(lease=lease) + + def test_poll_imds_report_ready_false(self, report_ready_func, + fake_resp, m_dhcp, m_net): + """The poll_imds should not call reporting ready + when flag is false""" + report_marker = self.tmp_path('report_marker', self.tmp) + write_file(report_marker, content='dont run report_ready :)') + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'unknown-245': '624c3620'}] + dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + mock_path = ( + 'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE') + with mock.patch(mock_path, report_marker): + dsa._poll_imds() + self.assertEqual(report_ready_func.call_count, 0) + + +@mock.patch('cloudinit.sources.DataSourceAzure.util.subp') +@mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') +@mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') +@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') +@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') +@mock.patch('requests.Session.request') +class TestAzureDataSourcePreprovisioning(CiTestCase): + + def setUp(self): + super(TestAzureDataSourcePreprovisioning, self).setUp() + tmp = self.tmp_dir() + self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) + self.paths = helpers.Paths({'cloud_dir': tmp}) + dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d + def test_poll_imds_returns_ovf_env(self, fake_resp, m_dhcp, m_net, - m_is_bsd, *args): + m_is_bsd, write_f, subp): """The _poll_imds method should return the ovf_env.xml.""" m_is_bsd.return_value = False m_dhcp.return_value = [{ @@ -1194,12 +1617,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): prefix_or_mask='255.255.255.0', router='192.168.2.1') self.assertEqual(m_net.call_count, 1) - @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') - @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') - @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') - @mock.patch('requests.Session.request') def test__reprovision_calls__poll_imds(self, fake_resp, m_dhcp, m_net, - m_is_bsd, *args): + m_is_bsd, write_f, subp): """The _reprovision method should call poll IMDS.""" m_is_bsd.return_value = False m_dhcp.return_value = [{ @@ -1216,7 +1635,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): fake_resp.return_value = mock.MagicMock(status_code=200, text=content, content=content) dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - md, ud, cfg, d = dsa._reprovision() + md, _ud, cfg, _d = dsa._reprovision() self.assertEqual(md['local-hostname'], hostname) self.assertEqual(cfg['system_info']['default_user']['name'], username) self.assertEqual(fake_resp.call_args_list, @@ -1231,32 +1650,95 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): prefix_or_mask='255.255.255.0', router='192.168.2.1') self.assertEqual(m_net.call_count, 1) - @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') - @mock.patch('os.path.isfile') - def test__should_reprovision_with_true_cfg(self, isfile, write_f, *args): - """The _should_reprovision method should return true with config - flag present.""" - isfile.return_value = False - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - self.assertTrue(dsa._should_reprovision( - (None, None, {'PreprovisionedVm': True}, None))) - @mock.patch('os.path.isfile') - def test__should_reprovision_with_file_existing(self, isfile, *args): - """The _should_reprovision method should return True if the sentinal - exists.""" - isfile.return_value = True - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - self.assertTrue(dsa._should_reprovision( - (None, None, {'preprovisionedvm': False}, None))) +class TestRemoveUbuntuNetworkConfigScripts(CiTestCase): - @mock.patch('os.path.isfile') - def test__should_reprovision_returns_false(self, isfile, *args): - """The _should_reprovision method should return False - if config and sentinal are not present.""" - isfile.return_value = False - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - self.assertFalse(dsa._should_reprovision((None, None, {}, None))) + with_logs = True + + def setUp(self): + super(TestRemoveUbuntuNetworkConfigScripts, self).setUp() + self.tmp = self.tmp_dir() + + def test_remove_network_scripts_removes_both_files_and_directories(self): + """Any files or directories in paths are removed when present.""" + file1 = self.tmp_path('file1', dir=self.tmp) + subdir = self.tmp_path('sub1', dir=self.tmp) + subfile = self.tmp_path('leaf1', dir=subdir) + write_file(file1, 'file1content') + write_file(subfile, 'leafcontent') + dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[subdir, file1]) + + for path in (file1, subdir, subfile): + self.assertFalse(os.path.exists(path), + 'Found unremoved: %s' % path) + + expected_logs = [ + 'INFO: Removing Ubuntu extended network scripts because cloud-init' + ' updates Azure network configuration on the following event:' + ' System boot.', + 'Recursively deleting %s' % subdir, + 'Attempting to remove %s' % file1] + for log in expected_logs: + self.assertIn(log, self.logs.getvalue()) + + def test_remove_network_scripts_only_attempts_removal_if_path_exists(self): + """Any files or directories absent are skipped without error.""" + dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[ + self.tmp_path('nodirhere/', dir=self.tmp), + self.tmp_path('notfilehere', dir=self.tmp)]) + self.assertNotIn('/not/a', self.logs.getvalue()) # No delete logs + + @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists') + def test_remove_network_scripts_default_removes_stock_scripts(self, + m_exists): + """Azure's stock ubuntu image scripts and artifacts are removed.""" + # Report path absent on all to avoid delete operation + m_exists.return_value = False + dsaz.maybe_remove_ubuntu_network_config_scripts() + calls = m_exists.call_args_list + for path in dsaz.UBUNTU_EXTENDED_NETWORK_SCRIPTS: + self.assertIn(mock.call(path), calls) + + +class TestWBIsPlatformViable(CiTestCase): + """White box tests for _is_platform_viable.""" + with_logs = True + + @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data') + def test_true_on_non_azure_chassis(self, m_read_dmi_data): + """Return True if DMI chassis-asset-tag is AZURE_CHASSIS_ASSET_TAG.""" + m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG + self.assertTrue(dsaz._is_platform_viable('doesnotmatter')) + + @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists') + @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data') + def test_true_on_azure_ovf_env_in_seed_dir(self, m_read_dmi_data, m_exist): + """Return True if ovf-env.xml exists in known seed dirs.""" + # Non-matching Azure chassis-asset-tag + m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X' + + m_exist.return_value = True + self.assertTrue(dsaz._is_platform_viable('/some/seed/dir')) + m_exist.called_once_with('/other/seed/dir') + + def test_false_on_no_matching_azure_criteria(self): + """Report non-azure on unmatched asset tag, ovf-env absent and no dev. + + Return False when the asset tag doesn't match Azure's static + AZURE_CHASSIS_ASSET_TAG, no ovf-env.xml files exist in known seed dirs + and no devices have a label starting with prefix 'rd_rdfe_'. + """ + self.assertFalse(wrap_and_call( + 'cloudinit.sources.DataSourceAzure', + {'os.path.exists': False, + # Non-matching Azure chassis-asset-tag + 'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X', + 'util.which': None}, + dsaz._is_platform_viable, 'doesnotmatter')) + self.assertIn( + "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( + dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'), + self.logs.getvalue()) # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b42b073f..26b2b93d 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -85,7 +85,9 @@ class TestFindEndpoint(CiTestCase): self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None)) - def test_latest_lease_used(self): + @mock.patch('cloudinit.sources.helpers.azure.util.is_FreeBSD') + def test_latest_lease_used(self, m_is_freebsd): + m_is_freebsd.return_value = False # To avoid hitting load_file encoded_addresses = ['5:4:3:2', '4:3:2:1'] file_content = '\n'.join([self._build_lease_content(encoded_address) for encoded_address in encoded_addresses]) @@ -195,7 +197,7 @@ class TestAzureEndpointHttpClient(CiTestCase): self.addCleanup(patches.close) self.read_file_or_url = patches.enter_context( - mock.patch.object(azure_helper.util, 'read_file_or_url')) + mock.patch.object(azure_helper.url_helper, 'read_file_or_url')) def test_non_secure_get(self): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index f6a59b6b..380ad1b5 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -42,6 +42,9 @@ class CepkoMock(Cepko): class DataSourceCloudSigmaTest(test_helpers.CiTestCase): def setUp(self): super(DataSourceCloudSigmaTest, self).setUp() + self.add_patch( + "cloudinit.sources.DataSourceCloudSigma.util.is_container", + "m_is_container", return_value=False) self.paths = helpers.Paths({'run_dir': self.tmp_dir()}) self.datasource = DataSourceCloudSigma.DataSourceCloudSigma( "", "", paths=self.paths) diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index ec333888..6b01a4ea 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -20,6 +20,7 @@ from cloudinit.sources import ( DataSourceNoCloud as NoCloud, DataSourceOpenNebula as OpenNebula, DataSourceOpenStack as OpenStack, + DataSourceOracle as Oracle, DataSourceOVF as OVF, DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, @@ -37,9 +38,12 @@ DEFAULT_LOCAL = [ IBMCloud.DataSourceIBMCloud, NoCloud.DataSourceNoCloud, OpenNebula.DataSourceOpenNebula, + Oracle.DataSourceOracle, OVF.DataSourceOVF, SmartOS.DataSourceSmartOS, Ec2.DataSourceEc2Local, + OpenStack.DataSourceOpenStackLocal, + Scaleway.DataSourceScaleway, ] DEFAULT_NETWORK = [ @@ -54,7 +58,6 @@ DEFAULT_NETWORK = [ NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, - Scaleway.DataSourceScaleway, ] diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 68400f22..231619c9 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -136,6 +136,7 @@ NETWORK_DATA_3 = { ] } +BOND_MAC = "fa:16:3e:b3:72:36" NETWORK_DATA_BOND = { "services": [ {"type": "dns", "address": "1.1.1.191"}, @@ -163,7 +164,7 @@ NETWORK_DATA_BOND = { {"bond_links": ["eth0", "eth1"], "bond_miimon": 100, "bond_mode": "4", "bond_xmit_hash_policy": "layer3+4", - "ethernet_mac_address": "0c:c4:7a:34:6e:3c", + "ethernet_mac_address": BOND_MAC, "id": "bond0", "type": "bond"}, {"ethernet_mac_address": "fa:16:3e:b3:72:30", "id": "vlan2", "type": "vlan", "vlan_id": 602, @@ -224,6 +225,9 @@ class TestConfigDriveDataSource(CiTestCase): def setUp(self): super(TestConfigDriveDataSource, self).setUp() + self.add_patch( + "cloudinit.sources.DataSourceConfigDrive.util.find_devs_with", + "m_find_devs_with", return_value=[]) self.tmp = self.tmp_dir() def test_ec2_metadata(self): @@ -642,7 +646,7 @@ class TestConvertNetworkData(CiTestCase): routes) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), self.tmp) + network_state.parse_net_config_data(ncfg), target=self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -664,7 +668,7 @@ class TestConvertNetworkData(CiTestCase): eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), self.tmp) + network_state.parse_net_config_data(ncfg), target=self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -688,6 +692,9 @@ class TestConvertNetworkData(CiTestCase): self.assertIn("auto oeth0", eni_rendering) self.assertIn("auto oeth1", eni_rendering) self.assertIn("auto bond0", eni_rendering) + # The bond should have the given mac address + pos = eni_rendering.find("auto bond0") + self.assertIn(BOND_MAC, eni_rendering[pos:]) def test_vlan(self): # light testing of vlan config conversion and eni rendering @@ -695,7 +702,7 @@ class TestConvertNetworkData(CiTestCase): known_macs=KNOWN_MACS) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), self.tmp) + network_state.parse_net_config_data(ncfg), target=self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index dff8b1ec..497e7610 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -191,7 +191,6 @@ def register_mock_metaserver(base_url, data): register(base_url, 'not found', status=404) def myreg(*argc, **kwargs): - # print("register_url(%s, %s)" % (argc, kwargs)) return httpretty.register_uri(httpretty.GET, *argc, **kwargs) register_helper(myreg, base_url, data) @@ -236,7 +235,6 @@ class TestEc2(test_helpers.HttprettyTestCase): return_value=platform_data) if md: - httpretty.HTTPretty.allow_net_connect = False all_versions = ( [ds.min_metadata_version] + ds.extended_metadata_versions) for version in all_versions: @@ -255,7 +253,6 @@ class TestEc2(test_helpers.HttprettyTestCase): register_mock_metaserver(instance_id_url, None) return ds - @httpretty.activate def test_network_config_property_returns_version_1_network_data(self): """network_config property returns network version 1 for metadata. @@ -288,7 +285,6 @@ class TestEc2(test_helpers.HttprettyTestCase): m_get_mac.return_value = mac1 self.assertEqual(expected, ds.network_config) - @httpretty.activate def test_network_config_property_set_dhcp4_on_private_ipv4(self): """network_config property configures dhcp4 on private ipv4 nics. @@ -330,7 +326,6 @@ class TestEc2(test_helpers.HttprettyTestCase): ds._network_config = {'cached': 'data'} self.assertEqual({'cached': 'data'}, ds.network_config) - @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp): """Refresh the network_config Ec2 cache if network key is absent. @@ -364,7 +359,6 @@ class TestEc2(test_helpers.HttprettyTestCase): 'type': 'physical'}]} self.assertEqual(expected, ds.network_config) - @httpretty.activate def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self): """get_instance-id gets DataSourceEc2Local.identity if not present. @@ -397,7 +391,6 @@ class TestEc2(test_helpers.HttprettyTestCase): ds.metadata = DEFAULT_METADATA self.assertEqual('my-identity-id', ds.get_instance_id()) - @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): """Valid platform data should return true with strict_id true.""" @@ -409,7 +402,6 @@ class TestEc2(test_helpers.HttprettyTestCase): self.assertTrue(ret) self.assertEqual(0, m_dhcp.call_count) - @httpretty.activate def test_valid_platform_with_strict_false(self): """Valid platform data should return true with strict_id false.""" ds = self._setup_ds( @@ -419,7 +411,6 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertTrue(ret) - @httpretty.activate def test_unknown_platform_with_strict_true(self): """Unknown platform data with strict_id true should return False.""" uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a' @@ -430,7 +421,6 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertFalse(ret) - @httpretty.activate def test_unknown_platform_with_strict_false(self): """Unknown platform data with strict_id false should return True.""" uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a' @@ -462,7 +452,6 @@ class TestEc2(test_helpers.HttprettyTestCase): ' not {0}'.format(platform_name)) self.assertIn(message, self.logs.getvalue()) - @httpretty.activate @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): """DataSourceEc2Local returns False on BSD. @@ -481,7 +470,6 @@ class TestEc2(test_helpers.HttprettyTestCase): "FreeBSD doesn't support running dhclient with -sf", self.logs.getvalue()) - @httpretty.activate @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.find_fallback_nic') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index eb3cec42..41176c6a 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -78,7 +78,6 @@ def _set_mock_metadata(gce_meta=None): return (404, headers, '') # reset is needed. https://github.com/gabrielfalcao/HTTPretty/issues/316 - httpretty.reset() httpretty.register_uri(httpretty.GET, MD_URL_RE, body=_request_callback) diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py index 621cfe49..e639ae47 100644 --- a/tests/unittests/test_datasource/test_ibmcloud.py +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -259,4 +259,54 @@ class TestReadMD(test_helpers.CiTestCase): ret['metadata']) +class TestIsIBMProvisioning(test_helpers.FilesystemMockingTestCase): + """Test the _is_ibm_provisioning method.""" + inst_log = "/root/swinstall.log" + prov_cfg = "/root/provisioningConfiguration.cfg" + boot_ref = "/proc/1/environ" + with_logs = True + + def _call_with_root(self, rootd): + self.reRoot(rootd) + return ibm._is_ibm_provisioning() + + def test_no_config(self): + """No provisioning config means not provisioning.""" + self.assertFalse(self._call_with_root(self.tmp_dir())) + + def test_config_only(self): + """A provisioning config without a log means provisioning.""" + rootd = self.tmp_dir() + test_helpers.populate_dir(rootd, {self.prov_cfg: "key=value"}) + self.assertTrue(self._call_with_root(rootd)) + + def test_config_with_old_log(self): + """A config with a log from previous boot is not provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", -30), + self.boot_ref: ("PWD=/", 0)} + test_helpers.populate_dir_with_ts(rootd, data) + self.assertFalse(self._call_with_root(rootd=rootd)) + self.assertIn("from previous boot", self.logs.getvalue()) + + def test_config_with_new_log(self): + """A config with a log from this boot is provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", 30), + self.boot_ref: ("PWD=/", 0)} + test_helpers.populate_dir_with_ts(rootd, data) + self.assertTrue(self._call_with_root(rootd=rootd)) + self.assertIn("from current boot", self.logs.getvalue()) + + def test_config_and_log_no_reference(self): + """If the config and log existed, but no reference, assume not.""" + rootd = self.tmp_dir() + test_helpers.populate_dir( + rootd, {self.prov_cfg: "key=value", self.inst_log: "log data\n"}) + self.assertFalse(self._call_with_root(rootd=rootd)) + self.assertIn("no reference file", self.logs.getvalue()) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 6e4031cf..c84d067e 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -53,7 +53,7 @@ class TestMAASDataSource(CiTestCase): my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) - ud, md, vd = DataSourceMAAS.read_maas_seed_dir(my_d) + ud, md, _vd = DataSourceMAAS.read_maas_seed_dir(my_d) self.assertEqual(userdata, ud) for key in ('instance-id', 'local-hostname'): @@ -149,7 +149,7 @@ class TestMAASDataSource(CiTestCase): 'meta-data/local-hostname': 'test-hostname', 'meta-data/vendor-data': yaml.safe_dump(expected_vd).encode(), } - ud, md, vd = self.mock_read_maas_seed_url( + _ud, md, vd = self.mock_read_maas_seed_url( valid, "http://example.com/foo") self.assertEqual(valid['meta-data/instance-id'], md['instance-id']) diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 70d50de4..21931eb7 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -25,6 +25,8 @@ class TestNoCloudDataSource(CiTestCase): self.mocks.enter_context( mock.patch.object(util, 'get_cmdline', return_value=self.cmdline)) + self.mocks.enter_context( + mock.patch.object(util, 'read_dmi_data', return_value=None)) def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} @@ -51,9 +53,6 @@ class TestNoCloudDataSource(CiTestCase): class PsuedoException(Exception): pass - def my_find_devs_with(*args, **kwargs): - raise PsuedoException - self.mocks.enter_context( mock.patch.object(util, 'find_devs_with', side_effect=PsuedoException)) diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ab42f344..61591017 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -43,6 +43,7 @@ DS_PATH = "cloudinit.sources.DataSourceOpenNebula" class TestOpenNebulaDataSource(CiTestCase): parsed_user = None + allowed_subp = ['bash'] def setUp(self): super(TestOpenNebulaDataSource, self).setUp() @@ -354,6 +355,412 @@ class TestOpenNebulaNetwork(unittest.TestCase): system_nics = ('eth0', 'ens3') + def test_context_devname(self): + """Verify context_devname correctly returns mac and name.""" + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH1_MAC': '02:00:0a:12:0f:0f', } + expected = { + '02:00:0a:12:01:01': 'ETH0', + '02:00:0a:12:0f:0f': 'ETH1', } + net = ds.OpenNebulaNetwork(context) + self.assertEqual(expected, net.context_devname) + + def test_get_nameservers(self): + """ + Verify get_nameservers('device') correctly returns DNS server addresses + and search domains. + """ + context = { + 'DNS': '1.2.3.8', + 'ETH0_DNS': '1.2.3.6 1.2.3.7', + 'ETH0_SEARCH_DOMAIN': 'example.com example.org', } + expected = { + 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'], + 'search': ['example.com', 'example.org']} + net = ds.OpenNebulaNetwork(context) + val = net.get_nameservers('eth0') + self.assertEqual(expected, val) + + def test_get_mtu(self): + """Verify get_mtu('device') correctly returns MTU size.""" + context = {'ETH0_MTU': '1280'} + net = ds.OpenNebulaNetwork(context) + val = net.get_mtu('eth0') + self.assertEqual('1280', val) + + def test_get_ip(self): + """Verify get_ip('device') correctly returns IPv4 address.""" + context = {'ETH0_IP': PUBLIC_IP} + net = ds.OpenNebulaNetwork(context) + val = net.get_ip('eth0', MACADDR) + self.assertEqual(PUBLIC_IP, val) + + def test_get_ip_emptystring(self): + """ + Verify get_ip('device') correctly returns IPv4 address. + It returns IP address created by MAC address if ETH0_IP has empty + string. + """ + context = {'ETH0_IP': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_ip('eth0', MACADDR) + self.assertEqual(IP_BY_MACADDR, val) + + def test_get_ip6(self): + """ + Verify get_ip6('device') correctly returns IPv6 address. + In this case, IPv6 address is Given by ETH0_IP6. + """ + context = { + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_ULA': '', } + expected = [IP6_GLOBAL] + net = ds.OpenNebulaNetwork(context) + val = net.get_ip6('eth0') + self.assertEqual(expected, val) + + def test_get_ip6_ula(self): + """ + Verify get_ip6('device') correctly returns IPv6 address. + In this case, IPv6 address is Given by ETH0_IP6_ULA. + """ + context = { + 'ETH0_IP6': '', + 'ETH0_IP6_ULA': IP6_ULA, } + expected = [IP6_ULA] + net = ds.OpenNebulaNetwork(context) + val = net.get_ip6('eth0') + self.assertEqual(expected, val) + + def test_get_ip6_dual(self): + """ + Verify get_ip6('device') correctly returns IPv6 address. + In this case, IPv6 addresses are Given by ETH0_IP6 and ETH0_IP6_ULA. + """ + context = { + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_ULA': IP6_ULA, } + expected = [IP6_GLOBAL, IP6_ULA] + net = ds.OpenNebulaNetwork(context) + val = net.get_ip6('eth0') + self.assertEqual(expected, val) + + def test_get_ip6_prefix(self): + """ + Verify get_ip6_prefix('device') correctly returns IPv6 prefix. + """ + context = {'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX} + net = ds.OpenNebulaNetwork(context) + val = net.get_ip6_prefix('eth0') + self.assertEqual(IP6_PREFIX, val) + + def test_get_ip6_prefix_emptystring(self): + """ + Verify get_ip6_prefix('device') correctly returns IPv6 prefix. + It returns default value '64' if ETH0_IP6_PREFIX_LENGTH has empty + string. + """ + context = {'ETH0_IP6_PREFIX_LENGTH': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_ip6_prefix('eth0') + self.assertEqual('64', val) + + def test_get_gateway(self): + """ + Verify get_gateway('device') correctly returns IPv4 default gateway + address. + """ + context = {'ETH0_GATEWAY': '1.2.3.5'} + net = ds.OpenNebulaNetwork(context) + val = net.get_gateway('eth0') + self.assertEqual('1.2.3.5', val) + + def test_get_gateway6(self): + """ + Verify get_gateway6('device') correctly returns IPv6 default gateway + address. + """ + context = {'ETH0_GATEWAY6': IP6_GW} + net = ds.OpenNebulaNetwork(context) + val = net.get_gateway6('eth0') + self.assertEqual(IP6_GW, val) + + def test_get_mask(self): + """ + Verify get_mask('device') correctly returns IPv4 subnet mask. + """ + context = {'ETH0_MASK': '255.255.0.0'} + net = ds.OpenNebulaNetwork(context) + val = net.get_mask('eth0') + self.assertEqual('255.255.0.0', val) + + def test_get_mask_emptystring(self): + """ + Verify get_mask('device') correctly returns IPv4 subnet mask. + It returns default value '255.255.255.0' if ETH0_MASK has empty string. + """ + context = {'ETH0_MASK': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_mask('eth0') + self.assertEqual('255.255.255.0', val) + + def test_get_network(self): + """ + Verify get_network('device') correctly returns IPv4 network address. + """ + context = {'ETH0_NETWORK': '1.2.3.0'} + net = ds.OpenNebulaNetwork(context) + val = net.get_network('eth0', MACADDR) + self.assertEqual('1.2.3.0', val) + + def test_get_network_emptystring(self): + """ + Verify get_network('device') correctly returns IPv4 network address. + It returns network address created by MAC address if ETH0_NETWORK has + empty string. + """ + context = {'ETH0_NETWORK': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_network('eth0', MACADDR) + self.assertEqual('10.18.1.0', val) + + def test_get_field(self): + """ + Verify get_field('device', 'name') returns *context* value. + """ + context = {'ETH9_DUMMY': 'DUMMY_VALUE'} + net = ds.OpenNebulaNetwork(context) + val = net.get_field('eth9', 'dummy') + self.assertEqual('DUMMY_VALUE', val) + + def test_get_field_withdefaultvalue(self): + """ + Verify get_field('device', 'name', 'default value') returns *context* + value. + """ + context = {'ETH9_DUMMY': 'DUMMY_VALUE'} + net = ds.OpenNebulaNetwork(context) + val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE') + self.assertEqual('DUMMY_VALUE', val) + + def test_get_field_withdefaultvalue_emptycontext(self): + """ + Verify get_field('device', 'name', 'default value') returns *default* + value if context value is empty string. + """ + context = {'ETH9_DUMMY': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE') + self.assertEqual('DEFAULT_VALUE', val) + + def test_get_field_emptycontext(self): + """ + Verify get_field('device', 'name') returns None if context value is + empty string. + """ + context = {'ETH9_DUMMY': ''} + net = ds.OpenNebulaNetwork(context) + val = net.get_field('eth9', 'dummy') + self.assertEqual(None, val) + + def test_get_field_nonecontext(self): + """ + Verify get_field('device', 'name') returns None if context value is + None. + """ + context = {'ETH9_DUMMY': None} + net = ds.OpenNebulaNetwork(context) + val = net.get_field('eth9', 'dummy') + self.assertEqual(None, val) + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_gateway(self, m_get_phys_by_mac): + """Test rendering with/without IPv4 gateway""" + self.maxDiff = None + # empty ETH0_GATEWAY + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_GATEWAY': '', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + # set ETH0_GATEWAY + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_GATEWAY': '1.2.3.5', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'gateway4': '1.2.3.5', + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_gateway6(self, m_get_phys_by_mac): + """Test rendering with/without IPv6 gateway""" + self.maxDiff = None + # empty ETH0_GATEWAY6 + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_GATEWAY6': '', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + # set ETH0_GATEWAY6 + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_GATEWAY6': IP6_GW, } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'gateway6': IP6_GW, + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_ipv6address(self, m_get_phys_by_mac): + """Test rendering with/without IPv6 address""" + self.maxDiff = None + # empty ETH0_IP6, ETH0_IP6_ULA, ETH0_IP6_PREFIX_LENGTH + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_IP6': '', + 'ETH0_IP6_ULA': '', + 'ETH0_IP6_PREFIX_LENGTH': '', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + # set ETH0_IP6, ETH0_IP6_ULA, ETH0_IP6_PREFIX_LENGTH + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_IP6': IP6_GLOBAL, + 'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX, + 'ETH0_IP6_ULA': IP6_ULA, } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [ + IP_BY_MACADDR + '/' + IP4_PREFIX, + IP6_GLOBAL + '/' + IP6_PREFIX, + IP6_ULA + '/' + IP6_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_dns(self, m_get_phys_by_mac): + """Test rendering with/without DNS server, search domain""" + self.maxDiff = None + # empty DNS, ETH0_DNS, ETH0_SEARCH_DOMAIN + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'DNS': '', + 'ETH0_DNS': '', + 'ETH0_SEARCH_DOMAIN': '', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + # set DNS, ETH0_DNS, ETH0_SEARCH_DOMAIN + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'DNS': '1.2.3.8', + 'ETH0_DNS': '1.2.3.6 1.2.3.7', + 'ETH0_SEARCH_DOMAIN': 'example.com example.org', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'nameservers': { + 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'], + 'search': ['example.com', 'example.org']}, + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_mtu(self, m_get_phys_by_mac): + """Test rendering with/without MTU""" + self.maxDiff = None + # empty ETH0_MTU + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_MTU': '', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + + # set ETH0_MTU + context = { + 'ETH0_MAC': '02:00:0a:12:01:01', + 'ETH0_MTU': '1280', } + for nic in self.system_nics: + expected = { + 'version': 2, + 'ethernets': { + nic: { + 'mtu': '1280', + 'match': {'macaddress': MACADDR}, + 'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}} + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context) + self.assertEqual(net.gen_conf(), expected) + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") def test_eth0(self, m_get_phys_by_mac): for nic in self.system_nics: @@ -395,7 +802,6 @@ class TestOpenNebulaNetwork(unittest.TestCase): 'match': {'macaddress': MACADDR}, 'addresses': [IP_BY_MACADDR + '/16'], 'gateway4': '1.2.3.5', - 'gateway6': None, 'nameservers': { 'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8']}}}} @@ -494,7 +900,6 @@ class TestOpenNebulaNetwork(unittest.TestCase): 'match': {'macaddress': MAC_1}, 'addresses': ['10.3.1.3/16'], 'gateway4': '10.3.0.1', - 'gateway6': None, 'nameservers': { 'addresses': ['10.3.1.2', '1.2.3.8'], 'search': [ diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 42c31554..a731f1ed 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -12,11 +12,11 @@ import re from cloudinit.tests import helpers as test_helpers from six.moves.urllib.parse import urlparse -from six import StringIO +from six import StringIO, text_type from cloudinit import helpers from cloudinit import settings -from cloudinit.sources import convert_vendordata +from cloudinit.sources import BrokenMetadata, convert_vendordata, UNSET from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources.helpers import openstack from cloudinit import util @@ -69,6 +69,8 @@ EC2_VERSIONS = [ 'latest', ] +MOCK_PATH = 'cloudinit.sources.DataSourceOpenStack.' + # TODO _register_uris should leverage test_ec2.register_mock_metaserver. def _register_uris(version, ec2_files, ec2_meta, os_files): @@ -129,13 +131,14 @@ def _read_metadata_service(): class TestOpenStackDataSource(test_helpers.HttprettyTestCase): + + with_logs = True VERSION = 'latest' def setUp(self): super(TestOpenStackDataSource, self).setUp() self.tmp = self.tmp_dir() - @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) f = _read_metadata_service() @@ -157,7 +160,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c', metadata.get('instance-id')) - @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) f = _read_metadata_service() @@ -168,7 +170,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual({}, f.get('ec2-metadata')) self.assertEqual(2, f.get('version')) - @hp.activate def test_bad_metadata(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -177,7 +178,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.NonReadable, _read_metadata_service) - @hp.activate def test_bad_uuid(self): os_files = copy.deepcopy(OS_FILES) os_meta = copy.deepcopy(OSTACK_META) @@ -186,9 +186,8 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) + self.assertRaises(BrokenMetadata, _read_metadata_service) - @hp.activate def test_userdata_empty(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -201,7 +200,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('userdata')) - @hp.activate def test_vendordata_empty(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -213,32 +211,32 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) - @hp.activate def test_vendordata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith('vendor_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) + self.assertRaises(BrokenMetadata, _read_metadata_service) - @hp.activate def test_metadata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith('meta_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) + self.assertRaises(BrokenMetadata, _read_metadata_service) - @hp.activate - def test_datasource(self): + @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + def test_datasource(self, m_dhcp): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, - None, - helpers.Paths({'run_dir': self.tmp})) + ds_os = ds.DataSourceOpenStack( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + mock_path = MOCK_PATH + 'detect_openstack' + with test_helpers.mock.patch(mock_path) as m_detect_os: + m_detect_os.return_value = True + found = ds_os.get_data() self.assertTrue(found) self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) @@ -250,8 +248,40 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(2, len(ds_os.files)) self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure) self.assertIsNone(ds_os.vendordata_raw) + m_dhcp.assert_not_called() @hp.activate + @test_helpers.mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') + @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + def test_local_datasource(self, m_dhcp, m_net): + """OpenStackLocal calls EphemeralDHCPNetwork and gets instance data.""" + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + ds_os_local = ds.DataSourceOpenStackLocal( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + ds_os_local._fallback_interface = 'eth9' # Monkey patch for dhcp + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'broadcast-address': '192.168.2.255'}] + + self.assertIsNone(ds_os_local.version) + mock_path = MOCK_PATH + 'detect_openstack' + with test_helpers.mock.patch(mock_path) as m_detect_os: + m_detect_os.return_value = True + found = ds_os_local.get_data() + self.assertTrue(found) + self.assertEqual(2, ds_os_local.version) + md = dict(ds_os_local.metadata) + md.pop('instance-id', None) + md.pop('local-hostname', None) + self.assertEqual(OSTACK_META, md) + self.assertEqual(EC2_META, ds_os_local.ec2_metadata) + self.assertEqual(USER_DATA, ds_os_local.userdata_raw) + self.assertEqual(2, len(ds_os_local.files)) + self.assertEqual(VENDOR_DATA, ds_os_local.vendordata_pure) + self.assertIsNone(ds_os_local.vendordata_raw) + m_dhcp.assert_called_with('eth9') + def test_bad_datasource_meta(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -262,11 +292,17 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({'run_dir': self.tmp})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + mock_path = MOCK_PATH + 'detect_openstack' + with test_helpers.mock.patch(mock_path) as m_detect_os: + m_detect_os.return_value = True + found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + self.assertIn( + 'InvalidMetaDataException: Broken metadata address' + ' http://169.254.169.25', + self.logs.getvalue()) - @hp.activate def test_no_datasource(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -281,11 +317,53 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + mock_path = MOCK_PATH + 'detect_openstack' + with test_helpers.mock.patch(mock_path) as m_detect_os: + m_detect_os.return_value = True + found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) - @hp.activate + def test_network_config_disabled_by_datasource_config(self): + """The network_config can be disabled from datasource config.""" + mock_path = MOCK_PATH + 'openstack.convert_net_json' + ds_os = ds.DataSourceOpenStack( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + ds_os.ds_cfg = {'apply_network_config': False} + sample_json = {'links': [{'ethernet_mac_address': 'mymac'}], + 'networks': [], 'services': []} + ds_os.network_json = sample_json # Ignore this content from metadata + with test_helpers.mock.patch(mock_path) as m_convert_json: + self.assertIsNone(ds_os.network_config) + m_convert_json.assert_not_called() + + def test_network_config_from_network_json(self): + """The datasource gets network_config from network_data.json.""" + mock_path = MOCK_PATH + 'openstack.convert_net_json' + example_cfg = {'version': 1, 'config': []} + ds_os = ds.DataSourceOpenStack( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + sample_json = {'links': [{'ethernet_mac_address': 'mymac'}], + 'networks': [], 'services': []} + ds_os.network_json = sample_json + with test_helpers.mock.patch(mock_path) as m_convert_json: + m_convert_json.return_value = example_cfg + self.assertEqual(example_cfg, ds_os.network_config) + self.assertIn( + 'network config provided via network_json', self.logs.getvalue()) + m_convert_json.assert_called_with(sample_json, known_macs=None) + + def test_network_config_cached(self): + """The datasource caches the network_config property.""" + mock_path = MOCK_PATH + 'openstack.convert_net_json' + example_cfg = {'version': 1, 'config': []} + ds_os = ds.DataSourceOpenStack( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + ds_os._network_config = example_cfg + with test_helpers.mock.patch(mock_path) as m_convert_json: + self.assertEqual(example_cfg, ds_os.network_config) + m_convert_json.assert_not_called() + def test_disabled_datasource(self): os_files = copy.deepcopy(OS_FILES) os_meta = copy.deepcopy(OSTACK_META) @@ -304,10 +382,42 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + mock_path = MOCK_PATH + 'detect_openstack' + with test_helpers.mock.patch(mock_path) as m_detect_os: + m_detect_os.return_value = True + found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + @hp.activate + def test_wb__crawl_metadata_does_not_persist(self): + """_crawl_metadata returns current metadata and does not cache.""" + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + ds_os = ds.DataSourceOpenStack( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + crawled_data = ds_os._crawl_metadata() + self.assertEqual(UNSET, ds_os.ec2_metadata) + self.assertIsNone(ds_os.userdata_raw) + self.assertEqual(0, len(ds_os.files)) + self.assertIsNone(ds_os.vendordata_raw) + self.assertEqual( + ['dsmode', 'ec2-metadata', 'files', 'metadata', 'networkdata', + 'userdata', 'vendordata', 'version'], + sorted(crawled_data.keys())) + self.assertEqual('local', crawled_data['dsmode']) + self.assertEqual(EC2_META, crawled_data['ec2-metadata']) + self.assertEqual(2, len(crawled_data['files'])) + md = copy.deepcopy(crawled_data['metadata']) + md.pop('instance-id') + md.pop('local-hostname') + self.assertEqual(OSTACK_META, md) + self.assertEqual( + json.loads(OS_FILES['openstack/latest/network_data.json']), + crawled_data['networkdata']) + self.assertEqual(USER_DATA, crawled_data['userdata']) + self.assertEqual(VENDOR_DATA, crawled_data['vendordata']) + self.assertEqual(2, crawled_data['version']) + class TestVendorDataLoading(test_helpers.TestCase): def cvj(self, data): @@ -339,4 +449,200 @@ class TestVendorDataLoading(test_helpers.TestCase): data = {'foo': 'bar', 'cloud-init': ['VD_1', 'VD_2']} self.assertEqual(self.cvj(data), data['cloud-init']) + +@test_helpers.mock.patch(MOCK_PATH + 'util.is_x86') +class TestDetectOpenStack(test_helpers.CiTestCase): + + def test_detect_openstack_non_intel_x86(self, m_is_x86): + """Return True on non-intel platforms because dmi isn't conclusive.""" + m_is_x86.return_value = False + self.assertTrue( + ds.detect_openstack(), 'Expected detect_openstack == True') + + @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env') + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_not_detect_openstack_intel_x86_ec2(self, m_dmi, m_proc_env, + m_is_x86): + """Return False on EC2 platforms.""" + m_is_x86.return_value = True + # No product_name in proc/1/environ + m_proc_env.return_value = {'HOME': '/'} + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'HVM domU' # Nothing 'openstackish' on EC2 + if dmi_key == 'chassis-asset-tag': + return '' # Empty string on EC2 + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertFalse( + ds.detect_openstack(), 'Expected detect_openstack == False on EC2') + m_proc_env.assert_called_with(1) + + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_intel_product_name_compute(self, m_dmi, + m_is_x86): + """Return True on OpenStack compute and nova instances.""" + m_is_x86.return_value = True + openstack_product_names = ['OpenStack Nova', 'OpenStack Compute'] + + for product_name in openstack_product_names: + m_dmi.return_value = product_name + self.assertTrue( + ds.detect_openstack(), 'Failed to detect_openstack') + + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_opentelekomcloud_chassis_asset_tag(self, m_dmi, + m_is_x86): + """Return True on OpenStack reporting OpenTelekomCloud asset-tag.""" + m_is_x86.return_value = True + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'HVM domU' # Nothing 'openstackish' on OpenTelekomCloud + if dmi_key == 'chassis-asset-tag': + return 'OpenTelekomCloud' + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertTrue( + ds.detect_openstack(), + 'Expected detect_openstack == True on OpenTelekomCloud') + + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_oraclecloud_chassis_asset_tag(self, m_dmi, + m_is_x86): + """Return True on OpenStack reporting Oracle cloud asset-tag.""" + m_is_x86.return_value = True + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'Standard PC (i440FX + PIIX, 1996)' # No match + if dmi_key == 'chassis-asset-tag': + return 'OracleCloud.com' + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertTrue( + ds.detect_openstack(accept_oracle=True), + 'Expected detect_openstack == True on OracleCloud.com') + self.assertFalse( + ds.detect_openstack(accept_oracle=False), + 'Expected detect_openstack == False.') + + @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env') + @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data') + def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env, + m_is_x86): + """Return True when nova product_name specified in /proc/1/environ.""" + m_is_x86.return_value = True + # Nova product_name in proc/1/environ + m_proc_env.return_value = { + 'HOME': '/', 'product_name': 'OpenStack Nova'} + + def fake_dmi_read(dmi_key): + if dmi_key == 'system-product-name': + return 'HVM domU' # Nothing 'openstackish' + if dmi_key == 'chassis-asset-tag': + return '' # Nothin 'openstackish' + assert False, 'Unexpected dmi read of %s' % dmi_key + + m_dmi.side_effect = fake_dmi_read + self.assertTrue( + ds.detect_openstack(), + 'Expected detect_openstack == True on OpenTelekomCloud') + m_proc_env.assert_called_with(1) + + +class TestMetadataReader(test_helpers.HttprettyTestCase): + """Test the MetadataReader.""" + burl = 'http://169.254.169.254/' + md_base = { + 'availability_zone': 'myaz1', + 'hostname': 'sm-foo-test.novalocal', + "keys": [{"data": PUBKEY, "name": "brickies", "type": "ssh"}], + 'launch_index': 0, + 'name': 'sm-foo-test', + 'public_keys': {'mykey': PUBKEY}, + 'project_id': '6a103f813b774b9fb15a4fcd36e1c056', + 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'} + + def register(self, path, body=None, status=200): + content = (body if not isinstance(body, text_type) + else body.encode('utf-8')) + hp.register_uri( + hp.GET, self.burl + "openstack" + path, status=status, + body=content) + + def register_versions(self, versions): + self.register("", '\n'.join(versions)) + self.register("/", '\n'.join(versions)) + + def register_version(self, version, data): + content = '\n'.join(sorted(data.keys())) + self.register(version, content) + self.register(version + "/", content) + for path, content in data.items(): + self.register("/%s/%s" % (version, path), content) + self.register("/%s/%s" % (version, path), content) + if 'user_data' not in data: + self.register("/%s/user_data" % version, "nodata", status=404) + + def test__find_working_version(self): + """Test a working version ignores unsupported.""" + unsup = "2016-11-09" + self.register_versions( + [openstack.OS_FOLSOM, openstack.OS_LIBERTY, unsup, + openstack.OS_LATEST]) + self.assertEqual( + openstack.OS_LIBERTY, + openstack.MetadataReader(self.burl)._find_working_version()) + + def test__find_working_version_uses_latest(self): + """'latest' should be used if no supported versions.""" + unsup1, unsup2 = ("2016-11-09", '2017-06-06') + self.register_versions([unsup1, unsup2, openstack.OS_LATEST]) + self.assertEqual( + openstack.OS_LATEST, + openstack.MetadataReader(self.burl)._find_working_version()) + + def test_read_v2_os_ocata(self): + """Validate return value of read_v2 for os_ocata data.""" + md = copy.deepcopy(self.md_base) + md['devices'] = [] + network_data = {'links': [], 'networks': [], 'services': []} + vendor_data = {} + vendor_data2 = {"static": {}} + + data = { + 'meta_data.json': json.dumps(md), + 'network_data.json': json.dumps(network_data), + 'vendor_data.json': json.dumps(vendor_data), + 'vendor_data2.json': json.dumps(vendor_data2), + } + + self.register_versions([openstack.OS_OCATA, openstack.OS_LATEST]) + self.register_version(openstack.OS_OCATA, data) + + mock_read_ec2 = test_helpers.mock.MagicMock( + return_value={'instance-id': 'unused-ec2'}) + expected_md = copy.deepcopy(md) + expected_md.update( + {'instance-id': md['uuid'], 'local-hostname': md['hostname']}) + expected = { + 'userdata': '', # Annoying, no user-data results in empty string. + 'version': 2, + 'metadata': expected_md, + 'vendordata': vendor_data, + 'networkdata': network_data, + 'ec2-metadata': mock_read_ec2.return_value, + 'files': {}, + } + reader = openstack.MetadataReader(self.burl) + reader._read_ec2_metadata = mock_read_ec2 + self.assertEqual(expected, reader.read_v2()) + self.assertEqual(1, mock_read_ec2.call_count) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index fc4eb36e..9d52eb99 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -124,7 +124,9 @@ class TestDatasourceOVF(CiTestCase): ds = self.datasource(sys_cfg={}, distro={}, paths=paths) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': None}, + {'util.read_dmi_data': None, + 'transport_iso9660': (False, None, None), + 'transport_vmware_guestd': (False, None, None)}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( @@ -138,7 +140,9 @@ class TestDatasourceOVF(CiTestCase): paths=paths) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', - {'util.read_dmi_data': 'vmware'}, + {'util.read_dmi_data': 'vmware', + 'transport_iso9660': (False, None, None), + 'transport_vmware_guestd': (False, None, None)}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 8dec06b1..c2bc7a00 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -176,12 +176,18 @@ class TestDataSourceScaleway(HttprettyTestCase): self.vendordata_url = \ DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url'] - @httpretty.activate + self.add_patch('cloudinit.sources.DataSourceScaleway.on_scaleway', + '_m_on_scaleway', return_value=True) + self.add_patch( + 'cloudinit.sources.DataSourceScaleway.net.find_fallback_nic', + '_m_find_fallback_nic', return_value='scalewaynic0') + + @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4') @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', get_source_address_adapter) @mock.patch('cloudinit.util.get_cmdline') @mock.patch('time.sleep', return_value=None) - def test_metadata_ok(self, sleep, m_get_cmdline): + def test_metadata_ok(self, sleep, m_get_cmdline, dhcpv4): """ get_data() returns metadata, user data and vendor data. """ @@ -212,12 +218,12 @@ class TestDataSourceScaleway(HttprettyTestCase): self.assertIsNone(self.datasource.region) self.assertEqual(sleep.call_count, 0) - @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4') @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', get_source_address_adapter) @mock.patch('cloudinit.util.get_cmdline') @mock.patch('time.sleep', return_value=None) - def test_metadata_404(self, sleep, m_get_cmdline): + def test_metadata_404(self, sleep, m_get_cmdline, dhcpv4): """ get_data() returns metadata, but no user data nor vendor data. """ @@ -236,12 +242,12 @@ class TestDataSourceScaleway(HttprettyTestCase): self.assertIsNone(self.datasource.get_vendordata_raw()) self.assertEqual(sleep.call_count, 0) - @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4') @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', get_source_address_adapter) @mock.patch('cloudinit.util.get_cmdline') @mock.patch('time.sleep', return_value=None) - def test_metadata_rate_limit(self, sleep, m_get_cmdline): + def test_metadata_rate_limit(self, sleep, m_get_cmdline, dhcpv4): """ get_data() is rate limited two times by the metadata API when fetching user data. @@ -265,3 +271,67 @@ class TestDataSourceScaleway(HttprettyTestCase): self.assertEqual(self.datasource.get_userdata_raw(), DataResponses.FAKE_USER_DATA) self.assertEqual(sleep.call_count, 2) + + @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic') + @mock.patch('cloudinit.util.get_cmdline') + def test_network_config_ok(self, m_get_cmdline, fallback_nic): + """ + network_config will only generate IPv4 config if no ipv6 data is + available in the metadata + """ + m_get_cmdline.return_value = 'scaleway' + fallback_nic.return_value = 'ens2' + self.datasource.metadata['ipv6'] = None + + netcfg = self.datasource.network_config + resp = {'version': 1, + 'config': [{ + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}]}] + } + self.assertEqual(netcfg, resp) + + @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic') + @mock.patch('cloudinit.util.get_cmdline') + def test_network_config_ipv6_ok(self, m_get_cmdline, fallback_nic): + """ + network_config will only generate IPv4/v6 configs if ipv6 data is + available in the metadata + """ + m_get_cmdline.return_value = 'scaleway' + fallback_nic.return_value = 'ens2' + self.datasource.metadata['ipv6'] = { + 'address': '2000:abc:4444:9876::42:999', + 'gateway': '2000:abc:4444:9876::42:000', + 'netmask': '127', + } + + netcfg = self.datasource.network_config + resp = {'version': 1, + 'config': [{ + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}, + {'type': 'static', + 'address': '2000:abc:4444:9876::42:999', + 'gateway': '2000:abc:4444:9876::42:000', + 'netmask': '127', } + ] + + }] + } + self.assertEqual(netcfg, resp) + + @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic') + @mock.patch('cloudinit.util.get_cmdline') + def test_network_config_existing(self, m_get_cmdline, fallback_nic): + """ + network_config() should return the same data if a network config + already exists + """ + m_get_cmdline.return_value = 'scaleway' + self.datasource._network_config = '0xdeadbeef' + + netcfg = self.datasource.network_config + self.assertEqual(netcfg, '0xdeadbeef') diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 88bae5f9..46d67b94 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -1,4 +1,5 @@ # Copyright (C) 2013 Canonical Ltd. +# Copyright (c) 2018, Joyent, Inc. # # Author: Ben Howard <ben.howard@canonical.com> # @@ -15,26 +16,40 @@ from __future__ import print_function from binascii import crc32 import json +import multiprocessing import os import os.path import re -import shutil +import signal import stat -import tempfile +import unittest2 import uuid from cloudinit import serial from cloudinit.sources import DataSourceSmartOS from cloudinit.sources.DataSourceSmartOS import ( - convert_smartos_network_data as convert_net) + convert_smartos_network_data as convert_net, + SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ, + identify_file) import six from cloudinit import helpers as c_helpers -from cloudinit.util import b64e +from cloudinit.util import ( + b64e, subp, ProcessExecutionError, which, write_file) -from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase +from cloudinit.tests.helpers import ( + CiTestCase, mock, FilesystemMockingTestCase, skipIf) + +try: + import serial as _pyserial + assert _pyserial # avoid pyflakes error F401: import unused + HAS_PYSERIAL = True +except ImportError: + HAS_PYSERIAL = False + +DSMOS = 'cloudinit.sources.DataSourceSmartOS' SDC_NICS = json.loads(""" [ { @@ -318,12 +333,19 @@ MOCK_RETURNS = { DMI_DATA_RETURN = 'smartdc' +# Useful for calculating the length of a frame body. A SUCCESS body will be +# followed by more characters or be one character less if SUCCESS with no +# payload. See Section 4.3 of https://eng.joyent.com/mdata/protocol.html. +SUCCESS_LEN = len('0123abcd SUCCESS ') +NOTFOUND_LEN = len('0123abcd NOTFOUND') + class PsuedoJoyentClient(object): def __init__(self, data=None): if data is None: data = MOCK_RETURNS.copy() self.data = data + self._is_open = False return def get(self, key, default=None, strip=False): @@ -344,39 +366,43 @@ class PsuedoJoyentClient(object): def exists(self): return True + def open_transport(self): + assert(not self._is_open) + self._is_open = True + + def close_transport(self): + assert(self._is_open) + self._is_open = False + class TestSmartOSDataSource(FilesystemMockingTestCase): + jmc_cfact = None + get_smartos_environ = None + def setUp(self): super(TestSmartOSDataSource, self).setUp() - dsmos = 'cloudinit.sources.DataSourceSmartOS' - patcher = mock.patch(dsmos + ".jmc_client_factory") - self.jmc_cfact = patcher.start() - self.addCleanup(patcher.stop) - patcher = mock.patch(dsmos + ".get_smartos_environ") - self.get_smartos_environ = patcher.start() - self.addCleanup(patcher.stop) - - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.paths = c_helpers.Paths( - {'cloud_dir': self.tmp, 'run_dir': self.tmp}) - - self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp') + self.add_patch(DSMOS + ".get_smartos_environ", "get_smartos_environ") + self.add_patch(DSMOS + ".jmc_client_factory", "jmc_cfact") + self.legacy_user_d = self.tmp_path('legacy_user_tmp') os.mkdir(self.legacy_user_d) - - self.orig_lud = DataSourceSmartOS.LEGACY_USER_D - DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d - - def tearDown(self): - DataSourceSmartOS.LEGACY_USER_D = self.orig_lud - super(TestSmartOSDataSource, self).tearDown() + self.add_patch(DSMOS + ".LEGACY_USER_D", "m_legacy_user_d", + autospec=False, new=self.legacy_user_d) + self.add_patch(DSMOS + ".identify_file", "m_identify_file", + return_value="text/plain") def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, sys_cfg=None, ds_cfg=None): self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) self.get_smartos_environ.return_value = mode + tmpd = self.tmp_dir() + dirs = {'cloud_dir': self.tmp_path('cloud_dir', tmpd), + 'run_dir': self.tmp_path('run_dir')} + for d in dirs.values(): + os.mkdir(d) + paths = c_helpers.Paths(dirs) + if sys_cfg is None: sys_cfg = {} @@ -385,7 +411,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): sys_cfg['datasource']['SmartOS'] = ds_cfg return DataSourceSmartOS.DataSourceSmartOS( - sys_cfg, distro=None, paths=self.paths) + sys_cfg, distro=None, paths=paths) def test_no_base64(self): ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} @@ -421,6 +447,34 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): self.assertEqual(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) + def test_hostname_if_no_sdc_hostname(self): + my_returns = MOCK_RETURNS.copy() + my_returns['sdc:hostname'] = 'sdc-' + my_returns['hostname'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['hostname'], + dsrc.metadata['local-hostname']) + + def test_sdc_hostname_if_no_hostname(self): + my_returns = MOCK_RETURNS.copy() + my_returns['sdc:hostname'] = 'sdc-' + my_returns['hostname'] + del my_returns['hostname'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['sdc:hostname'], + dsrc.metadata['local-hostname']) + + def test_sdc_uuid_if_no_hostname_or_sdc_hostname(self): + my_returns = MOCK_RETURNS.copy() + del my_returns['hostname'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['sdc:uuid'], + dsrc.metadata['local-hostname']) + def test_userdata(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() @@ -445,6 +499,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d + print("legacy_script_f=%s" % legacy_script_f) self.assertTrue(os.path.exists(legacy_script_f)) self.assertTrue(os.path.islink(legacy_script_f)) user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] @@ -592,8 +647,68 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) +class TestIdentifyFile(CiTestCase): + """Test the 'identify_file' utility.""" + @skipIf(not which("file"), "command 'file' not available.") + def test_file_happy_path(self): + """Test file is available and functional on plain text.""" + fname = self.tmp_path("myfile") + write_file(fname, "plain text content here\n") + with self.allow_subp(["file"]): + self.assertEqual("text/plain", identify_file(fname)) + + @mock.patch(DSMOS + ".util.subp") + def test_returns_none_on_error(self, m_subp): + """On 'file' execution error, None should be returned.""" + m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99) + fname = self.tmp_path("myfile") + write_file(fname, "plain text content here\n") + self.assertEqual(None, identify_file(fname)) + self.assertEqual( + [mock.call(["file", "--brief", "--mime-type", fname])], + m_subp.call_args_list) + + +class ShortReader(object): + """Implements a 'read' interface for bytes provided. + much like io.BytesIO but the 'endbyte' acts as if EOF. + When it is reached a short will be returned.""" + def __init__(self, initial_bytes, endbyte=b'\0'): + self.data = initial_bytes + self.index = 0 + self.len = len(self.data) + self.endbyte = endbyte + + @property + def emptied(self): + return self.index >= self.len + + def read(self, size=-1): + """Read size bytes but not past a null.""" + if size == 0 or self.index >= self.len: + return b'' + + rsize = size + if size < 0 or size + self.index > self.len: + rsize = self.len - self.index + + next_null = self.data.find(self.endbyte, self.index, rsize) + if next_null >= 0: + rsize = next_null - self.index + 1 + i = self.index + self.index += rsize + ret = self.data[i:i + rsize] + if len(ret) and ret[-1:] == self.endbyte: + ret = ret[:-1] + return ret + + class TestJoyentMetadataClient(FilesystemMockingTestCase): + invalid = b'invalid command\n' + failure = b'FAILURE\n' + v2_ok = b'V2_OK\n' + def setUp(self): super(TestJoyentMetadataClient, self).setUp() @@ -603,7 +718,7 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): self.response_parts = { 'command': 'SUCCESS', 'crc': 'b5a9ff00', - 'length': 17 + len(b64e(self.metadata_value)), + 'length': SUCCESS_LEN + len(b64e(self.metadata_value)), 'payload': b64e(self.metadata_value), 'request_id': '{0:08x}'.format(self.request_id), } @@ -636,6 +751,11 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): return DataSourceSmartOS.JoyentMetadataClient( fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) + def _get_serial_client(self): + self.serial.timeout = 1 + return DataSourceSmartOS.JoyentMetadataSerialClient(None, + fp=self.serial) + def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), "{0} does not end with '{1}'".format( @@ -646,12 +766,14 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): "{0} does not start with '{1}'".format( repr(haystack), prefix)) + def assertNoMoreSideEffects(self, obj): + self.assertRaises(StopIteration, obj) + def test_get_metadata_writes_a_single_line(self): client = self._get_client() client.get('some_key') self.assertEqual(1, self.serial.write.call_count) written_line = self.serial.write.call_args[0][0] - print(type(written_line)) self.assertEndsWith(written_line.decode('ascii'), b'\n'.decode('ascii')) self.assertEqual(1, written_line.count(b'\n')) @@ -732,13 +854,75 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): def test_get_metadata_returns_None_if_value_not_found(self): self.response_parts['payload'] = '' self.response_parts['command'] = 'NOTFOUND' - self.response_parts['length'] = 17 + self.response_parts['length'] = NOTFOUND_LEN client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertIsNone(client.get('some_key')) + def test_negotiate(self): + client = self._get_client() + reader = ShortReader(self.v2_ok) + client.fp.read.side_effect = reader.read + client._negotiate() + self.assertTrue(reader.emptied) + + def test_negotiate_short_response(self): + client = self._get_client() + # chopped '\n' from v2_ok. + reader = ShortReader(self.v2_ok[:-1] + b'\0') + client.fp.read.side_effect = reader.read + self.assertRaises(DataSourceSmartOS.JoyentMetadataTimeoutException, + client._negotiate) + self.assertTrue(reader.emptied) + + def test_negotiate_bad_response(self): + client = self._get_client() + reader = ShortReader(b'garbage\n' + self.v2_ok) + client.fp.read.side_effect = reader.read + self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, + client._negotiate) + self.assertEqual(self.v2_ok, client.fp.read()) + + def test_serial_open_transport(self): + client = self._get_serial_client() + reader = ShortReader(b'garbage\0' + self.invalid + self.v2_ok) + client.fp.read.side_effect = reader.read + client.open_transport() + self.assertTrue(reader.emptied) + + def test_flush_failure(self): + client = self._get_serial_client() + reader = ShortReader(b'garbage' + b'\0' + self.failure + + self.invalid + self.v2_ok) + client.fp.read.side_effect = reader.read + client.open_transport() + self.assertTrue(reader.emptied) + + def test_flush_many_timeouts(self): + client = self._get_serial_client() + reader = ShortReader(b'\0' * 100 + self.invalid + self.v2_ok) + client.fp.read.side_effect = reader.read + client.open_transport() + self.assertTrue(reader.emptied) + + def test_list_metadata_returns_list(self): + parts = ['foo', 'bar'] + value = b64e('\n'.join(parts)) + self.response_parts['payload'] = value + self.response_parts['crc'] = '40873553' + self.response_parts['length'] = SUCCESS_LEN + len(value) + client = self._get_client() + self.assertEqual(client.list(), parts) + + def test_list_metadata_returns_empty_list_if_no_customer_metadata(self): + del self.response_parts['payload'] + self.response_parts['length'] = SUCCESS_LEN - 1 + self.response_parts['crc'] = '14e563ba' + client = self._get_client() + self.assertEqual(client.list(), []) + -class TestNetworkConversion(TestCase): +class TestNetworkConversion(CiTestCase): def test_convert_simple(self): expected = { 'version': 1, @@ -872,4 +1056,94 @@ class TestNetworkConversion(TestCase): found = convert_net(SDC_NICS_SINGLE_GATEWAY) self.assertEqual(expected, found) + def test_routes_on_all_nics(self): + routes = [ + {'linklocal': False, 'dst': '3.0.0.0/8', 'gateway': '8.12.42.3'}, + {'linklocal': False, 'dst': '4.0.0.0/8', 'gateway': '10.210.1.4'}] + expected = { + 'version': 1, + 'config': [ + {'mac_address': '90:b8:d0:d8:82:b4', 'mtu': 1500, + 'name': 'net0', 'type': 'physical', + 'subnets': [{'address': '8.12.42.26/24', + 'gateway': '8.12.42.1', 'type': 'static', + 'routes': [{'network': '3.0.0.0/8', + 'gateway': '8.12.42.3'}, + {'network': '4.0.0.0/8', + 'gateway': '10.210.1.4'}]}]}, + {'mac_address': '90:b8:d0:0a:51:31', 'mtu': 1500, + 'name': 'net1', 'type': 'physical', + 'subnets': [{'address': '10.210.1.27/24', 'type': 'static', + 'routes': [{'network': '3.0.0.0/8', + 'gateway': '8.12.42.3'}, + {'network': '4.0.0.0/8', + 'gateway': '10.210.1.4'}]}]}]} + found = convert_net(SDC_NICS_SINGLE_GATEWAY, routes=routes) + self.maxDiff = None + self.assertEqual(expected, found) + + +@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, + "Only supported on KVM and bhyve guests under SmartOS") +@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), + "Requires write access to " + SERIAL_DEVICE) +@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available") +class TestSerialConcurrency(CiTestCase): + """ + This class tests locking on an actual serial port, and as such can only + be run in a kvm or bhyve guest running on a SmartOS host. A test run on + a metadata socket will not be valid because a metadata socket ensures + there is only one session over a connection. In contrast, in the + absence of proper locking multiple processes opening the same serial + port can corrupt each others' exchanges with the metadata server. + + This takes on the order of 2 to 3 minutes to run. + """ + allowed_subp = ['mdata-get'] + + def setUp(self): + self.mdata_proc = multiprocessing.Process(target=self.start_mdata_loop) + self.mdata_proc.start() + super(TestSerialConcurrency, self).setUp() + + def tearDown(self): + # os.kill() rather than mdata_proc.terminate() to avoid console spam. + os.kill(self.mdata_proc.pid, signal.SIGKILL) + self.mdata_proc.join() + super(TestSerialConcurrency, self).tearDown() + + def start_mdata_loop(self): + """ + The mdata-get command is repeatedly run in a separate process so + that it may try to race with metadata operations performed in the + main test process. Use of mdata-get is better than two processes + using the protocol implementation in DataSourceSmartOS because we + are testing to be sure that cloud-init and mdata-get respect each + others locks. + """ + rcs = list(range(0, 256)) + while True: + subp(['mdata-get', 'sdc:routes'], rcs=rcs) + + def test_all_keys(self): + self.assertIsNotNone(self.mdata_proc.pid) + ds = DataSourceSmartOS + keys = [tup[0] for tup in ds.SMARTOS_ATTRIB_MAP.values()] + keys.extend(ds.SMARTOS_ATTRIB_JSON.values()) + + client = ds.jmc_client_factory(smartos_type=SMARTOS_ENV_KVM) + self.assertIsNotNone(client) + + # The behavior that we are testing for was observed mdata-get running + # 10 times at roughly the same time as cloud-init fetched each key + # once. cloud-init would regularly see failures before making it + # through all keys once. + for _ in range(0, 3): + for key in keys: + # We don't care about the return value, just that it doesn't + # thrown any exceptions. + client.get(key) + + self.assertIsNone(self.mdata_proc.exitcode) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 5670904a..c3f258d5 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -1,7 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +import re + from cloudinit import distros -from cloudinit.tests.helpers import (TestCase, mock) +from cloudinit import ssh_util +from cloudinit.tests.helpers import (CiTestCase, mock) class MyBaseDistro(distros.Distro): @@ -44,8 +47,12 @@ class MyBaseDistro(distros.Distro): @mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False) @mock.patch("cloudinit.distros.util.subp") -class TestCreateUser(TestCase): +class TestCreateUser(CiTestCase): + + with_logs = True + def setUp(self): + super(TestCreateUser, self).setUp() self.dist = MyBaseDistro() def _useradd2call(self, args): @@ -145,4 +152,92 @@ class TestCreateUser(TestCase): mock.call(['passwd', '-l', user])] self.assertEqual(m_subp.call_args_list, expected) + def test_explicit_sudo_false(self, m_subp, m_is_snappy): + user = 'foouser' + self.dist.create_user(user, sudo=False) + self.assertEqual( + m_subp.call_args_list, + [self._useradd2call([user, '-m']), + mock.call(['passwd', '-l', user])]) + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_setup_ssh_authorized_keys_with_string( + self, m_setup_user_keys, m_subp, m_is_snappy): + """ssh_authorized_keys allows string and calls setup_user_keys.""" + user = 'foouser' + self.dist.create_user(user, ssh_authorized_keys='mykey') + self.assertEqual( + m_subp.call_args_list, + [self._useradd2call([user, '-m']), + mock.call(['passwd', '-l', user])]) + m_setup_user_keys.assert_called_once_with(set(['mykey']), user) + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_setup_ssh_authorized_keys_with_list( + self, m_setup_user_keys, m_subp, m_is_snappy): + """ssh_authorized_keys allows lists and calls setup_user_keys.""" + user = 'foouser' + self.dist.create_user(user, ssh_authorized_keys=['key1', 'key2']) + self.assertEqual( + m_subp.call_args_list, + [self._useradd2call([user, '-m']), + mock.call(['passwd', '-l', user])]) + m_setup_user_keys.assert_called_once_with(set(['key1', 'key2']), user) + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_setup_ssh_authorized_keys_with_integer( + self, m_setup_user_keys, m_subp, m_is_snappy): + """ssh_authorized_keys warns on non-iterable/string type.""" + user = 'foouser' + self.dist.create_user(user, ssh_authorized_keys=-1) + m_setup_user_keys.assert_called_once_with(set([]), user) + match = re.match( + r'.*WARNING: Invalid type \'<(type|class) \'int\'>\' detected for' + ' \'ssh_authorized_keys\'.*', + self.logs.getvalue(), + re.DOTALL) + self.assertIsNotNone( + match, 'Missing ssh_authorized_keys invalid type warning') + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_create_user_with_ssh_redirect_user_no_cloud_keys( + self, m_setup_user_keys, m_subp, m_is_snappy): + """Log a warning when trying to redirect a user no cloud ssh keys.""" + user = 'foouser' + self.dist.create_user(user, ssh_redirect_user='someuser') + self.assertIn( + 'WARNING: Unable to disable ssh logins for foouser given ' + 'ssh_redirect_user: someuser. No cloud public-keys present.\n', + self.logs.getvalue()) + m_setup_user_keys.assert_not_called() + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_create_user_with_ssh_redirect_user_with_cloud_keys( + self, m_setup_user_keys, m_subp, m_is_snappy): + """Disable ssh when ssh_redirect_user and cloud ssh keys are set.""" + user = 'foouser' + self.dist.create_user( + user, ssh_redirect_user='someuser', cloud_public_ssh_keys=['key1']) + disable_prefix = ssh_util.DISABLE_USER_OPTS + disable_prefix = disable_prefix.replace('$USER', 'someuser') + disable_prefix = disable_prefix.replace('$DISABLE_USER', user) + m_setup_user_keys.assert_called_once_with( + set(['key1']), 'foouser', options=disable_prefix) + + @mock.patch('cloudinit.ssh_util.setup_user_keys') + def test_create_user_with_ssh_redirect_user_does_not_disable_auth_keys( + self, m_setup_user_keys, m_subp, m_is_snappy): + """Do not disable ssh_authorized_keys when ssh_redirect_user is set.""" + user = 'foouser' + self.dist.create_user( + user, ssh_authorized_keys='auth1', ssh_redirect_user='someuser', + cloud_public_ssh_keys=['key1']) + disable_prefix = ssh_util.DISABLE_USER_OPTS + disable_prefix = disable_prefix.replace('$USER', 'someuser') + disable_prefix = disable_prefix.replace('$DISABLE_USER', user) + self.assertEqual( + m_setup_user_keys.call_args_list, + [mock.call(set(['auth1']), user), # not disabled + mock.call(set(['key1']), 'foouser', options=disable_prefix)]) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 1c2e45fe..6e339355 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -2,24 +2,19 @@ import os from six import StringIO -import stat from textwrap import dedent try: from unittest import mock except ImportError: import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers -from cloudinit.net import eni from cloudinit import settings -from cloudinit.tests.helpers import FilesystemMockingTestCase +from cloudinit.tests.helpers import ( + FilesystemMockingTestCase, dir2dict, populate_dir) from cloudinit import util @@ -39,6 +34,19 @@ auto eth1 iface eth1 inet dhcp ''' +BASE_NET_CFG_FROM_V2 = ''' +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.1.5/24 + gateway 192.168.1.254 + +auto eth1 +iface eth1 inet dhcp +''' + BASE_NET_CFG_IPV6 = ''' auto lo iface lo inet loopback @@ -82,7 +90,7 @@ V1_NET_CFG = {'config': [{'name': 'eth0', 'type': 'physical'}], 'version': 1} -V1_NET_CFG_OUTPUT = """ +V1_NET_CFG_OUTPUT = """\ # This file is generated from information provided by # the datasource. Changes to it will not persist across an instance. # To disable cloud-init's network configuration capabilities, write a file @@ -116,7 +124,7 @@ V1_NET_CFG_IPV6 = {'config': [{'name': 'eth0', 'version': 1} -V1_TO_V2_NET_CFG_OUTPUT = """ +V1_TO_V2_NET_CFG_OUTPUT = """\ # This file is generated from information provided by # the datasource. Changes to it will not persist across an instance. # To disable cloud-init's network configuration capabilities, write a file @@ -145,7 +153,7 @@ V2_NET_CFG = { } -V2_TO_V2_NET_CFG_OUTPUT = """ +V2_TO_V2_NET_CFG_OUTPUT = """\ # This file is generated from information provided by # the datasource. Changes to it will not persist across an instance. # To disable cloud-init's network configuration capabilities, write a file @@ -176,18 +184,13 @@ class WriteBuffer(object): return self.buffer.getvalue() -class TestNetCfgDistro(FilesystemMockingTestCase): +class TestNetCfgDistroBase(FilesystemMockingTestCase): - frbsd_ifout = """\ -hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 - options=51b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,TSO4,LRO> - ether 00:15:5d:4c:73:00 - inet6 fe80::215:5dff:fe4c:7300%hn0 prefixlen 64 scopeid 0x2 - inet 10.156.76.127 netmask 0xfffffc00 broadcast 10.156.79.255 - nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL> - media: Ethernet autoselect (10Gbase-T <full-duplex>) - status: active -""" + def setUp(self): + super(TestNetCfgDistroBase, self).setUp() + self.add_patch('cloudinit.util.system_is_snappy', 'm_snappy') + self.add_patch('cloudinit.util.system_info', 'm_sysinfo') + self.m_sysinfo.return_value = {'dist': ('Distro', '99.1', 'Codename')} def _get_distro(self, dname, renderers=None): cls = distros.fetch(dname) @@ -198,144 +201,6 @@ hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 paths = helpers.Paths({}) return cls(dname, cfg.get('system_info'), paths) - def test_simple_write_ub(self): - ub_distro = self._get_distro('ubuntu') - with ExitStack() as mocks: - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - - ub_distro.apply_network(BASE_NET_CFG, False) - - self.assertEqual(len(write_bufs), 1) - eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' - self.assertIn(eni_name, write_bufs) - write_buf = write_bufs[eni_name] - self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) - self.assertEqual(write_buf.mode, 0o644) - - def test_apply_network_config_eni_ub(self): - ub_distro = self._get_distro('ubuntu') - with ExitStack() as mocks: - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - # eni availability checks - mocks.enter_context( - mock.patch.object(util, 'which', return_value=True)) - mocks.enter_context( - mock.patch.object(eni, 'available', return_value=True)) - mocks.enter_context( - mock.patch.object(util, 'ensure_dir')) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - mocks.enter_context( - mock.patch("cloudinit.net.eni.glob.glob", - return_value=[])) - - ub_distro.apply_network_config(V1_NET_CFG, False) - - self.assertEqual(len(write_bufs), 2) - eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' - self.assertIn(eni_name, write_bufs) - write_buf = write_bufs[eni_name] - self.assertEqual(str(write_buf).strip(), V1_NET_CFG_OUTPUT.strip()) - self.assertEqual(write_buf.mode, 0o644) - - def test_apply_network_config_v1_to_netplan_ub(self): - renderers = ['netplan'] - devlist = ['eth0', 'lo'] - ub_distro = self._get_distro('ubuntu', renderers=renderers) - with ExitStack() as mocks: - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - mocks.enter_context( - mock.patch.object(util, 'which', return_value=True)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'ensure_dir')) - mocks.enter_context( - mock.patch.object(util, 'subp', return_value=(0, 0))) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - mocks.enter_context( - mock.patch("cloudinit.net.netplan.get_devicelist", - return_value=devlist)) - - ub_distro.apply_network_config(V1_NET_CFG, False) - - self.assertEqual(len(write_bufs), 1) - netplan_name = '/etc/netplan/50-cloud-init.yaml' - self.assertIn(netplan_name, write_bufs) - write_buf = write_bufs[netplan_name] - self.assertEqual(str(write_buf).strip(), - V1_TO_V2_NET_CFG_OUTPUT.strip()) - self.assertEqual(write_buf.mode, 0o644) - - def test_apply_network_config_v2_passthrough_ub(self): - renderers = ['netplan'] - devlist = ['eth0', 'lo'] - ub_distro = self._get_distro('ubuntu', renderers=renderers) - with ExitStack() as mocks: - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - mocks.enter_context( - mock.patch.object(util, 'which', return_value=True)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'ensure_dir')) - mocks.enter_context( - mock.patch.object(util, 'subp', return_value=(0, 0))) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - # FreeBSD does not have '/sys/class/net' file, - # so we need mock here. - mocks.enter_context( - mock.patch.object(os, 'listdir', return_value=devlist)) - ub_distro.apply_network_config(V2_NET_CFG, False) - - self.assertEqual(len(write_bufs), 1) - netplan_name = '/etc/netplan/50-cloud-init.yaml' - self.assertIn(netplan_name, write_bufs) - write_buf = write_bufs[netplan_name] - self.assertEqual(str(write_buf).strip(), - V2_TO_V2_NET_CFG_OUTPUT.strip()) - self.assertEqual(write_buf.mode, 0o644) - def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) @@ -347,6 +212,20 @@ hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 for (k, v) in b1.items(): self.assertEqual(v, b2[k]) + +class TestNetCfgDistroFreebsd(TestNetCfgDistroBase): + + frbsd_ifout = """\ +hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 + options=51b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,TSO4,LRO> + ether 00:15:5d:4c:73:00 + inet6 fe80::215:5dff:fe4c:7300%hn0 prefixlen 64 scopeid 0x2 + inet 10.156.76.127 netmask 0xfffffc00 broadcast 10.156.79.255 + nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL> + media: Ethernet autoselect (10Gbase-T <full-duplex>) + status: active +""" + @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_list') @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_ifname_out') def test_get_ip_nic_freebsd(self, ifname_out, iflist): @@ -370,349 +249,59 @@ hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 res = frbsd_distro.generate_fallback_config() self.assertIsNotNone(res) - def test_simple_write_rh(self): - rh_distro = self._get_distro('rhel') - - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - with ExitStack() as mocks: - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', return_value='')) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - - rh_distro.apply_network(BASE_NET_CFG, False) - - self.assertEqual(len(write_bufs), 4) - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] - expected_buf = ''' -DEVICE="lo" -ONBOOT=yes -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] - expected_buf = ''' -DEVICE="eth0" -BOOTPROTO="static" -NETMASK="255.255.255.0" -IPADDR="192.168.1.5" -ONBOOT=yes -GATEWAY="192.168.1.254" -BROADCAST="192.168.1.0" -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] - expected_buf = ''' -DEVICE="eth1" -BOOTPROTO="dhcp" -ONBOOT=yes -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network', write_bufs) - write_buf = write_bufs['/etc/sysconfig/network'] - expected_buf = ''' -# Created by cloud-init v. 0.7 -NETWORKING=yes -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - def test_apply_network_config_rh(self): - renderers = ['sysconfig'] - rh_distro = self._get_distro('rhel', renderers=renderers) - - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - with ExitStack() as mocks: - # sysconfig availability checks - mocks.enter_context( - mock.patch.object(util, 'which', return_value=True)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', return_value='')) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=True)) - - rh_distro.apply_network_config(V1_NET_CFG, False) - - self.assertEqual(len(write_bufs), 5) - - # eth0 - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] - expected_buf = ''' -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=none -DEFROUTE=yes -DEVICE=eth0 -GATEWAY=192.168.1.254 -IPADDR=192.168.1.5 -NETMASK=255.255.255.0 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - # eth1 - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] - expected_buf = ''' -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=dhcp -DEVICE=eth1 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network', write_bufs) - write_buf = write_bufs['/etc/sysconfig/network'] - expected_buf = ''' -# Created by cloud-init v. 0.7 -NETWORKING=yes -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - def test_write_ipv6_rhel(self): - rh_distro = self._get_distro('rhel') - - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - with ExitStack() as mocks: - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', return_value='')) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=False)) - rh_distro.apply_network(BASE_NET_CFG_IPV6, False) - - self.assertEqual(len(write_bufs), 4) - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] - expected_buf = ''' -DEVICE="lo" -ONBOOT=yes -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] - expected_buf = ''' -DEVICE="eth0" -BOOTPROTO="static" -NETMASK="255.255.255.0" -IPADDR="192.168.1.5" -ONBOOT=yes -GATEWAY="192.168.1.254" -BROADCAST="192.168.1.0" -IPV6INIT=yes -IPV6ADDR="2607:f0d0:1002:0011::2" -IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] - expected_buf = ''' -DEVICE="eth1" -BOOTPROTO="static" -NETMASK="255.255.255.0" -IPADDR="192.168.1.6" -ONBOOT=no -GATEWAY="192.168.1.254" -BROADCAST="192.168.1.0" -IPV6INIT=yes -IPV6ADDR="2607:f0d0:1002:0011::3" -IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network', write_bufs) - write_buf = write_bufs['/etc/sysconfig/network'] - expected_buf = ''' -# Created by cloud-init v. 0.7 -NETWORKING=yes -NETWORKING_IPV6=yes -IPV6_AUTOCONF=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - def test_apply_network_config_ipv6_rh(self): - renderers = ['sysconfig'] - rh_distro = self._get_distro('rhel', renderers=renderers) - - write_bufs = {} - - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - with ExitStack() as mocks: - mocks.enter_context( - mock.patch.object(util, 'which', return_value=True)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', return_value='')) - mocks.enter_context( - mock.patch.object(os.path, 'isfile', return_value=True)) - - rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) - - self.assertEqual(len(write_bufs), 5) - - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] - expected_buf = ''' -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=none -DEFROUTE=yes -DEVICE=eth0 -IPV6ADDR=2607:f0d0:1002:0011::2/64 -IPV6INIT=yes -IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', - write_bufs) - write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] - expected_buf = ''' -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=dhcp -DEVICE=eth1 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - - self.assertIn('/etc/sysconfig/network', write_bufs) - write_buf = write_bufs['/etc/sysconfig/network'] - expected_buf = ''' -# Created by cloud-init v. 0.7 -NETWORKING=yes -NETWORKING_IPV6=yes -IPV6_AUTOCONF=no -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) - def test_simple_write_freebsd(self): fbsd_distro = self._get_distro('freebsd') - write_bufs = {} + rc_conf = '/etc/rc.conf' read_bufs = { - '/etc/rc.conf': '', - '/etc/resolv.conf': '', + rc_conf: 'initial-rc-conf-not-validated', + '/etc/resolv.conf': 'initial-resolv-conf-not-validated', } - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - def replace_read(fname, read_cb=None, quiet=False): - if fname not in read_bufs: - if fname in write_bufs: - return str(write_bufs[fname]) - raise IOError("%s not found" % fname) - else: - if fname in write_bufs: - return str(write_bufs[fname]) - return read_bufs[fname] - - with ExitStack() as mocks: - mocks.enter_context( - mock.patch.object(util, 'subp', return_value=('vtnet0', ''))) - mocks.enter_context( - mock.patch.object(os.path, 'exists', return_value=False)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', replace_read)) - - fbsd_distro.apply_network(BASE_NET_CFG, False) - - self.assertIn('/etc/rc.conf', write_bufs) - write_buf = write_bufs['/etc/rc.conf'] - expected_buf = ''' -ifconfig_vtnet0="192.168.1.5 netmask 255.255.255.0" -ifconfig_vtnet1="DHCP" -defaultrouter="192.168.1.254" -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) + tmpd = self.tmp_dir() + populate_dir(tmpd, read_bufs) + with self.reRooted(tmpd): + with mock.patch("cloudinit.distros.freebsd.util.subp", + return_value=('vtnet0', '')): + fbsd_distro.apply_network(BASE_NET_CFG, False) + results = dir2dict(tmpd) + + self.assertIn(rc_conf, results) + self.assertCfgEquals( + dedent('''\ + ifconfig_vtnet0="192.168.1.5 netmask 255.255.255.0" + ifconfig_vtnet1="DHCP" + defaultrouter="192.168.1.254" + '''), results[rc_conf]) + self.assertEqual(0o644, get_mode(rc_conf, tmpd)) + + def test_simple_write_freebsd_from_v2eni(self): + fbsd_distro = self._get_distro('freebsd') + + rc_conf = '/etc/rc.conf' + read_bufs = { + rc_conf: 'initial-rc-conf-not-validated', + '/etc/resolv.conf': 'initial-resolv-conf-not-validated', + } - def test_apply_network_config_fallback(self): + tmpd = self.tmp_dir() + populate_dir(tmpd, read_bufs) + with self.reRooted(tmpd): + with mock.patch("cloudinit.distros.freebsd.util.subp", + return_value=('vtnet0', '')): + fbsd_distro.apply_network(BASE_NET_CFG_FROM_V2, False) + results = dir2dict(tmpd) + + self.assertIn(rc_conf, results) + self.assertCfgEquals( + dedent('''\ + ifconfig_vtnet0="192.168.1.5 netmask 255.255.255.0" + ifconfig_vtnet1="DHCP" + defaultrouter="192.168.1.254" + '''), results[rc_conf]) + self.assertEqual(0o644, get_mode(rc_conf, tmpd)) + + def test_apply_network_config_fallback_freebsd(self): fbsd_distro = self._get_distro('freebsd') # a weak attempt to verify that we don't have an implementation @@ -729,89 +318,293 @@ defaultrouter="192.168.1.254" "subnets": [{"type": "dhcp"}]}], 'version': 1} - write_bufs = {} + rc_conf = '/etc/rc.conf' read_bufs = { - '/etc/rc.conf': '', - '/etc/resolv.conf': '', + rc_conf: 'initial-rc-conf-not-validated', + '/etc/resolv.conf': 'initial-resolv-conf-not-validated', } - def replace_write(filename, content, mode=0o644, omode="wb"): - buf = WriteBuffer() - buf.mode = mode - buf.omode = omode - buf.write(content) - write_bufs[filename] = buf - - def replace_read(fname, read_cb=None, quiet=False): - if fname not in read_bufs: - if fname in write_bufs: - return str(write_bufs[fname]) - raise IOError("%s not found" % fname) - else: - if fname in write_bufs: - return str(write_bufs[fname]) - return read_bufs[fname] - - with ExitStack() as mocks: - mocks.enter_context( - mock.patch.object(util, 'subp', return_value=('vtnet0', ''))) - mocks.enter_context( - mock.patch.object(os.path, 'exists', return_value=False)) - mocks.enter_context( - mock.patch.object(util, 'write_file', replace_write)) - mocks.enter_context( - mock.patch.object(util, 'load_file', replace_read)) - - fbsd_distro.apply_network_config(mynetcfg, bring_up=False) - - self.assertIn('/etc/rc.conf', write_bufs) - write_buf = write_bufs['/etc/rc.conf'] - expected_buf = ''' -ifconfig_vtnet0="DHCP" -''' - self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEqual(write_buf.mode, 0o644) + tmpd = self.tmp_dir() + populate_dir(tmpd, read_bufs) + with self.reRooted(tmpd): + with mock.patch("cloudinit.distros.freebsd.util.subp", + return_value=('vtnet0', '')): + fbsd_distro.apply_network_config(mynetcfg, bring_up=False) + results = dir2dict(tmpd) + + self.assertIn(rc_conf, results) + self.assertCfgEquals('ifconfig_vtnet0="DHCP"', results[rc_conf]) + self.assertEqual(0o644, get_mode(rc_conf, tmpd)) + + +class TestNetCfgDistroUbuntuEni(TestNetCfgDistroBase): + + def setUp(self): + super(TestNetCfgDistroUbuntuEni, self).setUp() + self.distro = self._get_distro('ubuntu', renderers=['eni']) + + def eni_path(self): + return '/etc/network/interfaces.d/50-cloud-init.cfg' + + def _apply_and_verify_eni(self, apply_fn, config, expected_cfgs=None, + bringup=False): + if not expected_cfgs: + raise ValueError('expected_cfg must not be None') + + tmpd = None + with mock.patch('cloudinit.net.eni.available') as m_avail: + m_avail.return_value = True + with self.reRooted(tmpd) as tmpd: + apply_fn(config, bringup) + + results = dir2dict(tmpd) + for cfgpath, expected in expected_cfgs.items(): + print("----------") + print(expected) + print("^^^^ expected | rendered VVVVVVV") + print(results[cfgpath]) + print("----------") + self.assertEqual(expected, results[cfgpath]) + self.assertEqual(0o644, get_mode(cfgpath, tmpd)) - def test_simple_write_opensuse(self): - """Opensuse network rendering writes appropriate sysconfg files.""" - tmpdir = self.tmp_dir() - self.patchOS(tmpdir) - self.patchUtils(tmpdir) - distro = self._get_distro('opensuse') + def test_apply_network_config_eni_ub(self): + expected_cfgs = { + self.eni_path(): V1_NET_CFG_OUTPUT, + } + # ub_distro.apply_network_config(V1_NET_CFG, False) + self._apply_and_verify_eni(self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy()) + + +class TestNetCfgDistroUbuntuNetplan(TestNetCfgDistroBase): + def setUp(self): + super(TestNetCfgDistroUbuntuNetplan, self).setUp() + self.distro = self._get_distro('ubuntu', renderers=['netplan']) + self.devlist = ['eth0', 'lo'] + + def _apply_and_verify_netplan(self, apply_fn, config, expected_cfgs=None, + bringup=False): + if not expected_cfgs: + raise ValueError('expected_cfg must not be None') + + tmpd = None + with mock.patch('cloudinit.net.netplan.available', + return_value=True): + with mock.patch("cloudinit.net.netplan.get_devicelist", + return_value=self.devlist): + with self.reRooted(tmpd) as tmpd: + apply_fn(config, bringup) + + results = dir2dict(tmpd) + for cfgpath, expected in expected_cfgs.items(): + print("----------") + print(expected) + print("^^^^ expected | rendered VVVVVVV") + print(results[cfgpath]) + print("----------") + self.assertEqual(expected, results[cfgpath]) + self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + + def netplan_path(self): + return '/etc/netplan/50-cloud-init.yaml' - distro.apply_network(BASE_NET_CFG, False) + def test_apply_network_config_v1_to_netplan_ub(self): + expected_cfgs = { + self.netplan_path(): V1_TO_V2_NET_CFG_OUTPUT, + } + + # ub_distro.apply_network_config(V1_NET_CFG, False) + self._apply_and_verify_netplan(self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy()) - lo_path = os.path.join(tmpdir, 'etc/sysconfig/network/ifcfg-lo') - eth0_path = os.path.join(tmpdir, 'etc/sysconfig/network/ifcfg-eth0') - eth1_path = os.path.join(tmpdir, 'etc/sysconfig/network/ifcfg-eth1') + def test_apply_network_config_v2_passthrough_ub(self): expected_cfgs = { - lo_path: dedent(''' - STARTMODE="auto" - USERCONTROL="no" - FIREWALL="no" - '''), - eth0_path: dedent(''' - BOOTPROTO="static" - BROADCAST="192.168.1.0" - GATEWAY="192.168.1.254" - IPADDR="192.168.1.5" - NETMASK="255.255.255.0" - STARTMODE="auto" - USERCONTROL="no" - ETHTOOL_OPTIONS="" - '''), - eth1_path: dedent(''' - BOOTPROTO="dhcp" - STARTMODE="auto" - USERCONTROL="no" - ETHTOOL_OPTIONS="" - ''') + self.netplan_path(): V2_TO_V2_NET_CFG_OUTPUT, } - for cfgpath in (lo_path, eth0_path, eth1_path): - self.assertCfgEquals( - expected_cfgs[cfgpath], - util.load_file(cfgpath)) - file_stat = os.stat(cfgpath) - self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) + # ub_distro.apply_network_config(V2_NET_CFG, False) + self._apply_and_verify_netplan(self.distro.apply_network_config, + V2_NET_CFG, + expected_cfgs=expected_cfgs.copy()) + + +class TestNetCfgDistroRedhat(TestNetCfgDistroBase): + + def setUp(self): + super(TestNetCfgDistroRedhat, self).setUp() + self.distro = self._get_distro('rhel', renderers=['sysconfig']) + + def ifcfg_path(self, ifname): + return '/etc/sysconfig/network-scripts/ifcfg-%s' % ifname + + def control_path(self): + return '/etc/sysconfig/network' + + def _apply_and_verify(self, apply_fn, config, expected_cfgs=None, + bringup=False): + if not expected_cfgs: + raise ValueError('expected_cfg must not be None') + + tmpd = None + with mock.patch('cloudinit.net.sysconfig.available') as m_avail: + m_avail.return_value = True + with self.reRooted(tmpd) as tmpd: + apply_fn(config, bringup) + + results = dir2dict(tmpd) + for cfgpath, expected in expected_cfgs.items(): + self.assertCfgEquals(expected, results[cfgpath]) + self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + + def test_apply_network_config_rh(self): + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0 + GATEWAY=192.168.1.254 + IPADDR=192.168.1.5 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('eth1'): dedent("""\ + BOOTPROTO=dhcp + DEVICE=eth1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.control_path(): dedent("""\ + NETWORKING=yes + """), + } + # rh_distro.apply_network_config(V1_NET_CFG, False) + self._apply_and_verify(self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy()) + + def test_apply_network_config_ipv6_rh(self): + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0 + IPV6ADDR=2607:f0d0:1002:0011::2/64 + IPV6INIT=yes + IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('eth1'): dedent("""\ + BOOTPROTO=dhcp + DEVICE=eth1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.control_path(): dedent("""\ + NETWORKING=yes + NETWORKING_IPV6=yes + IPV6_AUTOCONF=no + """), + } + # rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) + self._apply_and_verify(self.distro.apply_network_config, + V1_NET_CFG_IPV6, + expected_cfgs=expected_cfgs.copy()) + + +class TestNetCfgDistroOpensuse(TestNetCfgDistroBase): + + def setUp(self): + super(TestNetCfgDistroOpensuse, self).setUp() + self.distro = self._get_distro('opensuse', renderers=['sysconfig']) + + def ifcfg_path(self, ifname): + return '/etc/sysconfig/network/ifcfg-%s' % ifname + + def _apply_and_verify(self, apply_fn, config, expected_cfgs=None, + bringup=False): + if not expected_cfgs: + raise ValueError('expected_cfg must not be None') + + tmpd = None + with mock.patch('cloudinit.net.sysconfig.available') as m_avail: + m_avail.return_value = True + with self.reRooted(tmpd) as tmpd: + apply_fn(config, bringup) + + results = dir2dict(tmpd) + for cfgpath, expected in expected_cfgs.items(): + self.assertCfgEquals(expected, results[cfgpath]) + self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + + def test_apply_network_config_opensuse(self): + """Opensuse uses apply_network_config and renders sysconfig""" + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0 + GATEWAY=192.168.1.254 + IPADDR=192.168.1.5 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('eth1'): dedent("""\ + BOOTPROTO=dhcp + DEVICE=eth1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + } + self._apply_and_verify(self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy()) + + def test_apply_network_config_ipv6_opensuse(self): + """Opensuse uses apply_network_config and renders sysconfig w/ipv6""" + expected_cfgs = { + self.ifcfg_path('eth0'): dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0 + IPV6ADDR=2607:f0d0:1002:0011::2/64 + IPV6INIT=yes + IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + self.ifcfg_path('eth1'): dedent("""\ + BOOTPROTO=dhcp + DEVICE=eth1 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + } + self._apply_and_verify(self.distro.apply_network_config, + V1_NET_CFG_IPV6, + expected_cfgs=expected_cfgs.copy()) + + +def get_mode(path, target=None): + return os.stat(util.target_path(target, path)).st_mode & 0o777 # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 0fa9cdb5..fa4b6cfe 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -22,6 +22,12 @@ bcfg = { class TestUGNormalize(TestCase): + def setUp(self): + super(TestUGNormalize, self).setUp() + self.add_patch('cloudinit.util.system_is_snappy', 'm_snappy') + self.add_patch('cloudinit.util.system_info', 'm_sysinfo') + self.m_sysinfo.return_value = {'dist': ('Distro', '99.1', 'Codename')} + def _make_distro(self, dtype, def_user=None): cfg = dict(settings.CFG_BUILTIN) cfg['system_info']['distro'] = dtype diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 53643989..46778e95 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +from collections import namedtuple import copy import os from uuid import uuid4 @@ -7,9 +8,11 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util from cloudinit.tests.helpers import ( - CiTestCase, dir2dict, populate_dir) + CiTestCase, dir2dict, populate_dir, populate_dir_with_ts) -from cloudinit.sources import DataSourceIBMCloud as dsibm +from cloudinit.sources import DataSourceIBMCloud as ds_ibm +from cloudinit.sources import DataSourceSmartOS as ds_smartos +from cloudinit.sources import DataSourceOracle as ds_oracle UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") @@ -66,19 +69,29 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor" P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" -IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg" IBM_CONFIG_UUID = "9796-932E" +MOCK_VIRT_IS_CONTAINER_OTHER = {'name': 'detect_virt', + 'RET': 'container-other', 'ret': 0} MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} +# currenty' SmartOS hypervisor "bhyve" is unknown by systemd-detect-virt. +MOCK_VIRT_IS_VM_OTHER = {'name': 'detect_virt', 'RET': 'vm-other', 'ret': 0} MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} +shell_true = 0 +shell_false = 1 -class TestDsIdentify(CiTestCase): +CallReturn = namedtuple('CallReturn', + ['rc', 'stdout', 'stderr', 'cfg', 'files']) + + +class DsIdentifyBase(CiTestCase): dsid_path = os.path.realpath('tools/ds-identify') + allowed_subp = ['sh'] - def call(self, rootd=None, mocks=None, args=None, files=None, + def call(self, rootd=None, mocks=None, func="main", args=None, files=None, policy_dmi=DI_DEFAULT_POLICY, policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI, ec2_strict_id=DI_EC2_STRICT_ID_DEFAULT): @@ -135,7 +148,7 @@ class TestDsIdentify(CiTestCase): mocklines.append(write_mock(d)) endlines = [ - 'main %s' % ' '.join(['"%s"' % s for s in args]) + func + ' ' + ' '.join(['"%s"' % s for s in args]) ] with open(wrap, "w") as fp: @@ -159,12 +172,14 @@ class TestDsIdentify(CiTestCase): cfg = {"_INVALID_YAML": contents, "_EXCEPTION": str(e)} - return rc, out, err, cfg, dir2dict(rootd) + return CallReturn(rc, out, err, cfg, dir2dict(rootd)) def _call_via_dict(self, data, rootd=None, **kwargs): # return output of self.call with a dict input like VALID_CFG[item] xwargs = {'rootd': rootd} - for k in ('mocks', 'args', 'policy_dmi', 'policy_no_dmi', 'files'): + passthrough = ('mocks', 'func', 'args', 'policy_dmi', + 'policy_no_dmi', 'files') + for k in passthrough: if k in data: xwargs[k] = data[k] if k in kwargs: @@ -178,18 +193,21 @@ class TestDsIdentify(CiTestCase): data, RC_FOUND, dslist=[data.get('ds'), DS_NONE]) def _check_via_dict(self, data, rc, dslist=None, **kwargs): - found_rc, out, err, cfg, files = self._call_via_dict(data, **kwargs) + ret = self._call_via_dict(data, **kwargs) good = False try: - self.assertEqual(rc, found_rc) + self.assertEqual(rc, ret.rc) if dslist is not None: - self.assertEqual(dslist, cfg['datasource_list']) + self.assertEqual(dslist, ret.cfg['datasource_list']) good = True finally: if not good: - _print_run_output(rc, out, err, cfg, files) - return rc, out, err, cfg, files + _print_run_output(ret.rc, ret.stdout, ret.stderr, ret.cfg, + ret.files) + return ret + +class TestDsIdentify(DsIdentifyBase): def test_wb_print_variables(self): """_print_info reports an array of discovered variables to stderr.""" data = VALID_CFG['Azure-dmi-detection'] @@ -237,20 +255,50 @@ class TestDsIdentify(CiTestCase): def test_config_drive(self): """ConfigDrive datasource has a disk with LABEL=config-2.""" self._test_ds_found('ConfigDrive') - return def test_config_drive_upper(self): """ConfigDrive datasource has a disk with LABEL=CONFIG-2.""" self._test_ds_found('ConfigDriveUpper') return + def test_config_drive_seed(self): + """Config Drive seed directory.""" + self._test_ds_found('ConfigDrive-seed') + + def test_config_drive_interacts_with_ibmcloud_config_disk(self): + """Verify ConfigDrive interaction with IBMCloud. + + If ConfigDrive is enabled and not IBMCloud, then ConfigDrive + should claim the ibmcloud 'config-2' disk. + If IBMCloud is enabled, then ConfigDrive should skip.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + cfgpath = 'etc/cloud/cloud.cfg.d/99_networklayer_common.cfg' + + # with list including IBMCloud, config drive should be not found. + files[cfgpath] = 'datasource_list: [ ConfigDrive, IBMCloud ]\n' + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ret.cfg.get('datasource_list'), ['IBMCloud', 'None']) + + # But if IBMCloud is not enabled, config drive should claim this. + files[cfgpath] = 'datasource_list: [ ConfigDrive, NoCloud ]\n' + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ret.cfg.get('datasource_list'), ['ConfigDrive', 'None']) + def test_ibmcloud_template_userdata_in_provisioning(self): """Template provisioned with user-data during provisioning stage. Template provisioning with user-data has METADATA disk, datasource should return not found.""" data = copy.deepcopy(VALID_CFG['IBMCloud-metadata']) - data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + # change the 'is_ibm_provisioning' mock to return 1 (false) + isprov_m = [m for m in data['mocks'] + if m["name"] == "is_ibm_provisioning"][0] + isprov_m['ret'] = shell_true return self._check_via_dict(data, RC_NOT_FOUND) def test_ibmcloud_template_userdata(self): @@ -265,7 +313,8 @@ class TestDsIdentify(CiTestCase): no disks attached. Datasource should return not found.""" data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks']) - data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + data['mocks'].append( + {'name': 'is_ibm_provisioning', 'ret': shell_true}) return self._check_via_dict(data, RC_NOT_FOUND) def test_ibmcloud_template_no_userdata(self): @@ -290,11 +339,42 @@ class TestDsIdentify(CiTestCase): break if not offset: raise ValueError("Expected to find 'blkid' mock, but did not.") - data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID, + data['mocks'][offset]['out'] = d['out'].replace(ds_ibm.IBM_CONFIG_UUID, "DEAD-BEEF") self._check_via_dict( data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE]) + def test_ibmcloud_with_nocloud_seed(self): + """NoCloud seed should be preferred over IBMCloud. + + A nocloud seed should be preferred over IBMCloud even if enabled. + Ubuntu 16.04 images have <vlc>/seed/nocloud-net. LP: #1766401.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + files.update(VALID_CFG['NoCloud-seed']['files']) + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ['NoCloud', 'IBMCloud', 'None'], + ret.cfg.get('datasource_list')) + + def test_ibmcloud_with_configdrive_seed(self): + """ConfigDrive seed should be preferred over IBMCloud. + + A ConfigDrive seed should be preferred over IBMCloud even if enabled. + Ubuntu 16.04 images have a fstab entry that mounts the + METADATA disk into <vlc>/seed/config_drive. LP: ##1766401.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + files.update(VALID_CFG['ConfigDrive-seed']['files']) + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ['ConfigDrive', 'IBMCloud', 'None'], + ret.cfg.get('datasource_list')) + def test_policy_disabled(self): """A Builtin policy of 'disabled' should return not found. @@ -445,6 +525,92 @@ class TestDsIdentify(CiTestCase): """Hetzner cloud is identified in sys_vendor.""" self._test_ds_found('Hetzner') + def test_smartos_bhyve(self): + """SmartOS cloud identified by SmartDC in dmi.""" + self._test_ds_found('SmartOS-bhyve') + + def test_smartos_lxbrand(self): + """SmartOS cloud identified on lxbrand container.""" + self._test_ds_found('SmartOS-lxbrand') + + def test_smartos_lxbrand_requires_socket(self): + """SmartOS cloud should not be identified if no socket file.""" + mycfg = copy.deepcopy(VALID_CFG['SmartOS-lxbrand']) + del mycfg['files'][ds_smartos.METADATA_SOCKFILE] + self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled") + + def test_path_env_gets_set_from_main(self): + """PATH environment should always have some tokens when main is run. + + We explicitly call main as we want to ensure it updates PATH.""" + cust = copy.deepcopy(VALID_CFG['NoCloud']) + rootd = self.tmp_dir() + mpp = 'main-printpath' + pre = "MYPATH=" + cust['files'][mpp] = ( + 'PATH="/mycust/path"; main; r=$?; echo ' + pre + '$PATH; exit $r;') + ret = self._check_via_dict( + cust, RC_FOUND, + func=".", args=[os.path.join(rootd, mpp)], rootd=rootd) + line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0] + toks = line.replace(pre, "").split(":") + expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"] + self.assertEqual(expected, [p for p in expected if p in toks], + "path did not have expected tokens") + + +class TestIsIBMProvisioning(DsIdentifyBase): + """Test the is_ibm_provisioning method in ds-identify.""" + + inst_log = "/root/swinstall.log" + prov_cfg = "/root/provisioningConfiguration.cfg" + boot_ref = "/proc/1/environ" + funcname = "is_ibm_provisioning" + + def test_no_config(self): + """No provisioning config means not provisioning.""" + ret = self.call(files={}, func=self.funcname) + self.assertEqual(shell_false, ret.rc) + + def test_config_only(self): + """A provisioning config without a log means provisioning.""" + ret = self.call(files={self.prov_cfg: "key=value"}, func=self.funcname) + self.assertEqual(shell_true, ret.rc) + + def test_config_with_old_log(self): + """A config with a log from previous boot is not provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", -30), + self.boot_ref: ("PWD=/", 0)} + populate_dir_with_ts(rootd, data) + ret = self.call(rootd=rootd, func=self.funcname) + self.assertEqual(shell_false, ret.rc) + self.assertIn("from previous boot", ret.stderr) + + def test_config_with_new_log(self): + """A config with a log from this boot is provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", 30), + self.boot_ref: ("PWD=/", 0)} + populate_dir_with_ts(rootd, data) + ret = self.call(rootd=rootd, func=self.funcname) + self.assertEqual(shell_true, ret.rc) + self.assertIn("from current boot", ret.stderr) + + +class TestOracle(DsIdentifyBase): + def test_found_by_chassis(self): + """Simple positive test of Oracle by chassis id.""" + self._test_ds_found('Oracle') + + def test_not_found(self): + """Simple negative test of Oracle.""" + mycfg = copy.deepcopy(VALID_CFG['Oracle']) + mycfg['files'][P_CHASSIS_ASSET_TAG] = "Not Oracle" + self._check_via_dict(mycfg, rc=RC_NOT_FOUND) + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" @@ -631,6 +797,12 @@ VALID_CFG = { }, ], }, + 'ConfigDrive-seed': { + 'ds': 'ConfigDrive', + 'files': { + os.path.join(P_SEED_DIR, 'config_drive', 'openstack', + 'latest', 'meta_data.json'): 'md\n'}, + }, 'Hetzner': { 'ds': 'Hetzner', 'files': {P_SYS_VENDOR: 'Hetzner\n'}, @@ -639,6 +811,7 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, @@ -652,12 +825,13 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(), 'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'}, {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2', - 'UUID': dsibm.IBM_CONFIG_UUID}, + 'UUID': ds_ibm.IBM_CONFIG_UUID}, {'DEVNAME': 'xvda2', 'TYPE': 'ext4', 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(), 'UUID': uuid4()}, @@ -669,6 +843,7 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, @@ -677,6 +852,38 @@ VALID_CFG = { }, ], }, + 'Oracle': { + 'ds': 'Oracle', + 'files': { + P_CHASSIS_ASSET_TAG: ds_oracle.CHASSIS_ASSET_TAG + '\n', + } + }, + 'SmartOS-bhyve': { + 'ds': 'SmartOS', + 'mocks': [ + MOCK_VIRT_IS_VM_OTHER, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'vda1', 'TYPE': 'ext4', + 'PARTUUID': '49ec635a-01'}, + {'DEVNAME': 'vda2', 'TYPE': 'swap', + 'LABEL': 'cloudimg-swap', 'PARTUUID': '49ec635a-02'}]), + }, + ], + 'files': {P_PRODUCT_NAME: 'SmartDC HVM\n'}, + }, + 'SmartOS-lxbrand': { + 'ds': 'SmartOS', + 'mocks': [ + MOCK_VIRT_IS_CONTAINER_OTHER, + {'name': 'uname', 'ret': 0, + 'out': ("Linux d43da87a-daca-60e8-e6d4-d2ed372662a3 4.3.0 " + "BrandZ virtual linux x86_64 GNU/Linux")}, + {'name': 'blkid', 'ret': 2, 'out': ''}, + ], + 'files': {ds_smartos.METADATA_SOCKFILE: 'would be a socket\n'}, + } + } # vi: ts=4 expandtab diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index af78997f..3f50f57d 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -11,7 +11,6 @@ from cloudinit import url_helper as uh class TestEc2Util(helpers.HttprettyTestCase): VERSION = 'latest' - @hp.activate def test_userdata_fetch(self): hp.register_uri(hp.GET, 'http://169.254.169.254/%s/user-data' % (self.VERSION), @@ -20,7 +19,6 @@ class TestEc2Util(helpers.HttprettyTestCase): userdata = eu.get_instance_userdata(self.VERSION) self.assertEqual('stuff', userdata.decode('utf-8')) - @hp.activate def test_userdata_fetch_fail_not_found(self): hp.register_uri(hp.GET, 'http://169.254.169.254/%s/user-data' % (self.VERSION), @@ -28,7 +26,6 @@ class TestEc2Util(helpers.HttprettyTestCase): userdata = eu.get_instance_userdata(self.VERSION, retries=0) self.assertEqual('', userdata) - @hp.activate def test_userdata_fetch_fail_server_dead(self): hp.register_uri(hp.GET, 'http://169.254.169.254/%s/user-data' % (self.VERSION), @@ -36,7 +33,6 @@ class TestEc2Util(helpers.HttprettyTestCase): userdata = eu.get_instance_userdata(self.VERSION, retries=0) self.assertEqual('', userdata) - @hp.activate def test_userdata_fetch_fail_server_not_found(self): hp.register_uri(hp.GET, 'http://169.254.169.254/%s/user-data' % (self.VERSION), @@ -44,7 +40,6 @@ class TestEc2Util(helpers.HttprettyTestCase): userdata = eu.get_instance_userdata(self.VERSION) self.assertEqual('', userdata) - @hp.activate def test_metadata_fetch_no_keys(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, @@ -62,7 +57,6 @@ class TestEc2Util(helpers.HttprettyTestCase): self.assertEqual(md['instance-id'], '123') self.assertEqual(md['ami-launch-index'], '1') - @hp.activate def test_metadata_fetch_key(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, @@ -83,7 +77,6 @@ class TestEc2Util(helpers.HttprettyTestCase): self.assertEqual(md['instance-id'], '123') self.assertEqual(1, len(md['public-keys'])) - @hp.activate def test_metadata_fetch_with_2_keys(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, @@ -108,7 +101,6 @@ class TestEc2Util(helpers.HttprettyTestCase): self.assertEqual(md['instance-id'], '123') self.assertEqual(2, len(md['public-keys'])) - @hp.activate def test_metadata_fetch_bdm(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, @@ -140,7 +132,6 @@ class TestEc2Util(helpers.HttprettyTestCase): self.assertEqual(bdm['ami'], 'sdb') self.assertEqual(bdm['ephemeral0'], 'sdc') - @hp.activate def test_metadata_no_security_credentials(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 6364d38e..e1a5d2c8 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -55,7 +55,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): return True def testMultiEmailIndex(self): - test_data = self.readResource('filter_cloud_multipart_2.email') + test_data = helpers.readResource('filter_cloud_multipart_2.email') ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) self.assertTrue(count_messages(message) > 0) @@ -70,7 +70,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): self.assertCounts(message, expected_counts) def testHeaderEmailIndex(self): - test_data = self.readResource('filter_cloud_multipart_header.email') + test_data = helpers.readResource('filter_cloud_multipart_header.email') ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) self.assertTrue(count_messages(message) > 0) @@ -85,7 +85,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): self.assertCounts(message, expected_counts) def testConfigEmailIndex(self): - test_data = self.readResource('filter_cloud_multipart_1.email') + test_data = helpers.readResource('filter_cloud_multipart_1.email') ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) self.assertTrue(count_messages(message) > 0) @@ -99,7 +99,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): self.assertCounts(message, expected_counts) def testNoneIndex(self): - test_data = self.readResource('filter_cloud_multipart.yaml') + test_data = helpers.readResource('filter_cloud_multipart.yaml') ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) start_count = count_messages(message) @@ -108,7 +108,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): self.assertTrue(self.equivalentMessage(message, filtered_message)) def testIndexes(self): - test_data = self.readResource('filter_cloud_multipart.yaml') + test_data = helpers.readResource('filter_cloud_multipart.yaml') ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) start_count = count_messages(message) diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 83f962a9..6a4b03ee 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -12,10 +12,6 @@ import shutil import tempfile -def load_tfile_or_url(*args, **kwargs): - return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)) - - class TestAptProxyConfig(TestCase): def setUp(self): super(TestAptProxyConfig, self).setUp() @@ -36,7 +32,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = load_tfile_or_url(self.pfile) + contents = util.load_file(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_http_proxy_written(self): @@ -46,7 +42,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = load_tfile_or_url(self.pfile) + contents = util.load_file(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_all_proxy_written(self): @@ -64,7 +60,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = load_tfile_or_url(self.pfile) + contents = util.load_file(self.pfile) for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) @@ -80,7 +76,7 @@ class TestAptProxyConfig(TestCase): cc_apt_configure.apply_apt_config({'proxy': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) - contents = load_tfile_or_url(self.pfile) + contents = util.load_file(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "foo")) def test_config_written(self): @@ -92,14 +88,14 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.cfile)) self.assertFalse(os.path.isfile(self.pfile)) - self.assertEqual(load_tfile_or_url(self.cfile), payload) + self.assertEqual(util.load_file(self.cfile), payload) def test_config_replaced(self): util.write_file(self.pfile, "content doesnt matter") cc_apt_configure.apply_apt_config({'conf': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.cfile)) - self.assertEqual(load_tfile_or_url(self.cfile), "foo") + self.assertEqual(util.load_file(self.cfile), "foo") def test_config_deleted(self): # if no 'conf' is provided, delete any previously written file diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index d2b96f0b..23bd6e10 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -64,13 +64,6 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted """) -def load_tfile_or_url(*args, **kwargs): - """load_tfile_or_url - load file and return content after decoding - """ - return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) - - class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """TestAptSourceConfigSourceList Main Class to test sources list rendering diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 46ca4ce4..a3132fbd 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -39,13 +39,6 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== ADD_APT_REPO_MATCH = r"^[\w-]+:\w" -def load_tfile_or_url(*args, **kwargs): - """load_tfile_or_url - load file and return content after decoding - """ - return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) - - class FakeDistro(object): """Fake Distro helper object""" def update_package_sources(self): @@ -125,7 +118,7 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://archive.ubuntu.com/ubuntu", "karmic-backports", @@ -157,13 +150,13 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, cfg) # extra verify on two extra files of this test - contents = load_tfile_or_url(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://archive.ubuntu.com/ubuntu", "precise-backports", "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://archive.ubuntu.com/ubuntu", "lucid-backports", @@ -220,7 +213,7 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "multiverse"), @@ -241,12 +234,12 @@ class TestAptSourceConfig(TestCase): # extra verify on two extra files of this test params = self._get_default_params() - contents = load_tfile_or_url(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "main"), contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "universe"), @@ -296,7 +289,7 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -336,14 +329,14 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile3} self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) - contents = load_tfile_or_url(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' 'cloud-init-test/ubuntu'), "xenial", "universe"), contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -375,7 +368,7 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 7bb1b7c4..90fe6eed 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -48,12 +48,9 @@ ADD_APT_REPO_MATCH = r"^[\w-]+:\w" TARGET = None - -def load_tfile(*args, **kwargs): - """load_tfile_or_url - load file and return content after decoding - """ - return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) +MOCK_LSB_RELEASE_DATA = { + 'id': 'Ubuntu', 'description': 'Ubuntu 18.04.1 LTS', + 'release': '18.04', 'codename': 'bionic'} class TestAptSourceConfig(t_help.FilesystemMockingTestCase): @@ -71,6 +68,9 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") self.join = os.path.join self.matcher = re.compile(ADD_APT_REPO_MATCH).search + self.add_patch( + 'cloudinit.config.cc_apt_configure.util.lsb_release', + 'm_lsb_release', return_value=MOCK_LSB_RELEASE_DATA.copy()) @staticmethod def _add_apt_sources(*args, **kwargs): @@ -83,7 +83,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): Get the most basic default mrror and release info to be used in tests """ params = {} - params['RELEASE'] = util.lsb_release()['codename'] + params['RELEASE'] = MOCK_LSB_RELEASE_DATA['release'] arch = 'amd64' params['MIRROR'] = cc_apt_configure.\ get_default_mirrors(arch)["PRIMARY"] @@ -119,7 +119,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://test.ubuntu.com/ubuntu", "karmic-backports", @@ -151,13 +151,13 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self._apt_src_basic(self.aptlistfile, cfg) # extra verify on two extra files of this test - contents = load_tfile(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://test.ubuntu.com/ubuntu", "precise-backports", "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - contents = load_tfile(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://test.ubuntu.com/ubuntu", "lucid-backports", @@ -174,7 +174,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "multiverse"), @@ -201,12 +201,12 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): # extra verify on two extra files of this test params = self._get_default_params() - contents = load_tfile(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "main"), contents, flags=re.IGNORECASE)) - contents = load_tfile(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "universe"), @@ -240,7 +240,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertTrue(os.path.isfile(filename)) - contents = load_tfile(filename) + contents = util.load_file(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -277,14 +277,14 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): 'keyid': "03683F77"}} self._apt_src_keyid(self.aptlistfile, cfg, 3) - contents = load_tfile(self.aptlistfile2) + contents = util.load_file(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' 'cloud-init-test/ubuntu'), "xenial", "universe"), contents, flags=re.IGNORECASE)) - contents = load_tfile(self.aptlistfile3) + contents = util.load_file(self.aptlistfile3) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -310,7 +310,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertTrue(os.path.isfile(self.aptlistfile)) - contents = load_tfile(self.aptlistfile) + contents = util.load_file(self.aptlistfile) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -471,7 +471,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): 'uri': 'http://testsec.ubuntu.com/%s/' % component}]} post = ("%s_dists_%s-updates_InRelease" % - (component, util.lsb_release()['codename'])) + (component, MOCK_LSB_RELEASE_DATA['codename'])) fromfn = ("%s/%s_%s" % (pre, archive, post)) tofn = ("%s/test.ubuntu.com_%s" % (pre, post)) @@ -528,7 +528,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): expected = sorted([npre + suff for opre, npre, suff in files]) # create files - for (opre, npre, suff) in files: + for (opre, _npre, suff) in files: fpath = os.path.join(apt_lists_d, opre + suff) util.write_file(fpath, content=fpath) @@ -949,7 +949,8 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""") self.assertEqual( orig, cc_apt_configure.disable_suites(["proposed"], orig, rel)) - def test_apt_v3_mirror_search_dns(self): + @mock.patch("cloudinit.util.get_hostname", return_value='abc.localdomain') + def test_apt_v3_mirror_search_dns(self, m_get_hostname): """test_apt_v3_mirror_search_dns - Test searching dns patterns""" pmir = "phit" smir = "shit" diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index 29fc25e4..a76760fa 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -1,9 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config import cc_bootcmd +from cloudinit.config.cc_bootcmd import handle, schema from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from cloudinit.tests.helpers import ( + CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) import logging import tempfile @@ -50,7 +51,7 @@ class TestBootcmd(CiTestCase): """When the provided config doesn't contain bootcmd, skip it.""" cfg = {} mycloud = self._get_cloud('ubuntu') - cc_bootcmd.handle('notimportant', cfg, mycloud, LOG, None) + handle('notimportant', cfg, mycloud, LOG, None) self.assertIn( "Skipping module named notimportant, no 'bootcmd' key", self.logs.getvalue()) @@ -60,7 +61,7 @@ class TestBootcmd(CiTestCase): invalid_config = {'bootcmd': 1} cc = self._get_cloud('ubuntu') with self.assertRaises(TypeError) as context_manager: - cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + handle('cc_bootcmd', invalid_config, cc, LOG, []) self.assertIn('Failed to shellify bootcmd', self.logs.getvalue()) self.assertEqual( "Input to shellify was type 'int'. Expected list or tuple.", @@ -76,7 +77,7 @@ class TestBootcmd(CiTestCase): invalid_config = {'bootcmd': 1} cc = self._get_cloud('ubuntu') with self.assertRaises(TypeError): - cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + handle('cc_bootcmd', invalid_config, cc, LOG, []) self.assertIn( 'Invalid config:\nbootcmd: 1 is not of type \'array\'', self.logs.getvalue()) @@ -93,7 +94,7 @@ class TestBootcmd(CiTestCase): 'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} cc = self._get_cloud('ubuntu') with self.assertRaises(TypeError) as context_manager: - cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + handle('cc_bootcmd', invalid_config, cc, LOG, []) expected_warnings = [ 'bootcmd.1: 20 is not valid under any of the given schemas', 'bootcmd.3: {\'a\': \'n\'} is not valid under any of the given' @@ -117,7 +118,8 @@ class TestBootcmd(CiTestCase): 'echo {0} $INSTANCE_ID > {1}'.format(my_id, out_file)]} with mock.patch(self._etmpfile_path, FakeExtendedTempFile): - cc_bootcmd.handle('cc_bootcmd', valid_config, cc, LOG, []) + with self.allow_subp(['/bin/sh']): + handle('cc_bootcmd', valid_config, cc, LOG, []) self.assertEqual(my_id + ' iid-datasource-none\n', util.load_file(out_file)) @@ -127,15 +129,33 @@ class TestBootcmd(CiTestCase): valid_config = {'bootcmd': ['exit 1']} # Script with error with mock.patch(self._etmpfile_path, FakeExtendedTempFile): - with self.assertRaises(util.ProcessExecutionError) as ctxt_manager: - cc_bootcmd.handle('does-not-matter', valid_config, cc, LOG, []) + with self.allow_subp(['/bin/sh']): + with self.assertRaises(util.ProcessExecutionError) as ctxt: + handle('does-not-matter', valid_config, cc, LOG, []) self.assertIn( 'Unexpected error while running command.\n' "Command: ['/bin/sh',", - str(ctxt_manager.exception)) + str(ctxt.exception)) self.assertIn( 'Failed to run bootcmd module does-not-matter', self.logs.getvalue()) +@skipUnlessJsonSchema() +class TestSchema(CiTestCase, SchemaTestCaseMixin): + """Directly test schema rather than through handle.""" + + schema = schema + + def test_duplicates_are_fine_array_array(self): + """Duplicated commands array/array entries are allowed.""" + self.assertSchemaValid( + ["byebye", "byebye"], 'command entries can be duplicate') + + def test_duplicates_are_fine_array_string(self): + """Duplicated commands array/string entries are allowed.""" + self.assertSchemaValid( + ["echo bye", "echo bye"], "command entries can be duplicate.") + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 0136a93d..b16532ea 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -14,27 +14,43 @@ from cloudinit.sources import DataSourceNone from cloudinit import util from cloudinit.tests.helpers import ( - CiTestCase, FilesystemMockingTestCase, mock, skipIf) + HttprettyTestCase, FilesystemMockingTestCase, mock, skipIf) LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) +# This is adjusted to use http because using with https causes issue +# in some openssl/httpretty combinations. +# https://github.com/gabrielfalcao/HTTPretty/issues/242 +# We saw issue in opensuse 42.3 with +# httpretty=0.8.8-7.1 ndg-httpsclient=0.4.0-3.2 pyOpenSSL=16.0.0-4.1 +OMNIBUS_URL_HTTP = cc_chef.OMNIBUS_URL.replace("https:", "http:") -class TestInstallChefOmnibus(CiTestCase): + +class TestInstallChefOmnibus(HttprettyTestCase): def setUp(self): + super(TestInstallChefOmnibus, self).setUp() self.new_root = self.tmp_dir() - @httpretty.activate + @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP) def test_install_chef_from_omnibus_runs_chef_url_content(self): - """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script.""" - chef_outfile = self.tmp_path('chef.out', self.new_root) - response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + """install_chef_from_omnibus calls subp_blob_in_tempfile.""" + response = b'#!/bin/bash\necho "Hi Mom"' httpretty.register_uri( httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) - cc_chef.install_chef_from_omnibus() - self.assertEqual('Hi Mom\n', util.load_file(chef_outfile)) + ret = (None, None) # stdout, stderr but capture=False + + with mock.patch("cloudinit.config.cc_chef.util.subp_blob_in_tempfile", + return_value=ret) as m_subp_blob: + cc_chef.install_chef_from_omnibus() + # admittedly whitebox, but assuming subp_blob_in_tempfile works + # this should be fine. + self.assertEqual( + [mock.call(blob=response, args=[], basename='chef-omnibus-install', + capture=False)], + m_subp_blob.call_args_list) @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') @@ -65,7 +81,7 @@ class TestInstallChefOmnibus(CiTestCase): expected_subp_kwargs, m_subp_blob.call_args_list[0][1]) - @httpretty.activate + @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP) @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py index ced05a8d..d854afcb 100644 --- a/tests/unittests/test_handler/test_handler_etc_hosts.py +++ b/tests/unittests/test_handler/test_handler_etc_hosts.py @@ -49,6 +49,7 @@ class TestHostsFile(t_help.FilesystemMockingTestCase): if '192.168.1.1\tblah.blah.us\tblah' not in contents: self.assertIsNone('Default etc/hosts content modified') + @t_help.skipUnlessJinja() def test_write_etc_hosts_suse_template(self): cfg = { 'manage_etc_hosts': 'template', diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index a2054980..2478ebc4 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -33,45 +33,56 @@ class TestLxd(t_help.CiTestCase): cc = cloud.Cloud(ds, paths, {}, d, None) return cc + @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_init(self, mock_util): + def test_lxd_init(self, mock_util, m_maybe_clean): cc = self._get_cloud('ubuntu') mock_util.which.return_value = True + m_maybe_clean.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) self.assertTrue(mock_util.which.called) - init_call = mock_util.subp.call_args_list[0][0][0] - self.assertEqual(init_call, - ['lxd', 'init', '--auto', - '--network-address=0.0.0.0', - '--storage-backend=zfs', - '--storage-pool=poolname']) + # no bridge config, so maybe_cleanup should not be called. + self.assertFalse(m_maybe_clean.called) + self.assertEqual( + [mock.call(['lxd', 'waitready', '--timeout=300']), + mock.call( + ['lxd', 'init', '--auto', '--network-address=0.0.0.0', + '--storage-backend=zfs', '--storage-pool=poolname'])], + mock_util.subp.call_args_list) + @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_install(self, mock_util): + def test_lxd_install(self, mock_util, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() mock_util.which.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) self.assertNotIn('WARN', self.logs.getvalue()) self.assertTrue(cc.distro.install_packages.called) + cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) + self.assertFalse(m_maybe_clean.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] self.assertEqual(sorted(install_pkg), ['lxd', 'zfs']) + @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_init_does_nothing(self, mock_util): + def test_no_init_does_nothing(self, mock_util, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) self.assertFalse(mock_util.subp.called) + self.assertFalse(m_maybe_clean.called) + @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_lxd_does_nothing(self, mock_util): + def test_no_lxd_does_nothing(self, mock_util, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) self.assertFalse(mock_util.subp.called) + self.assertFalse(m_maybe_clean.called) def test_lxd_debconf_new_full(self): data = {"mode": "new", @@ -147,14 +158,13 @@ class TestLxd(t_help.CiTestCase): "domain": "lxd"} self.assertEqual( cc_lxd.bridge_to_cmd(data), - (["lxc", "network", "create", "testbr0", + (["network", "create", "testbr0", "ipv4.address=10.0.8.1/24", "ipv4.nat=true", "ipv4.dhcp.ranges=10.0.8.2-10.0.8.254", "ipv6.address=fd98:9e0:3744::1/64", - "ipv6.nat=true", "dns.domain=lxd", - "--force-local"], - ["lxc", "network", "attach-profile", - "testbr0", "default", "eth0", "--force-local"])) + "ipv6.nat=true", "dns.domain=lxd"], + ["network", "attach-profile", + "testbr0", "default", "eth0"])) def test_lxd_cmd_new_partial(self): data = {"mode": "new", @@ -163,19 +173,18 @@ class TestLxd(t_help.CiTestCase): "ipv6_nat": "true"} self.assertEqual( cc_lxd.bridge_to_cmd(data), - (["lxc", "network", "create", "lxdbr0", "ipv4.address=none", - "ipv6.address=fd98:9e0:3744::1/64", "ipv6.nat=true", - "--force-local"], - ["lxc", "network", "attach-profile", - "lxdbr0", "default", "eth0", "--force-local"])) + (["network", "create", "lxdbr0", "ipv4.address=none", + "ipv6.address=fd98:9e0:3744::1/64", "ipv6.nat=true"], + ["network", "attach-profile", + "lxdbr0", "default", "eth0"])) def test_lxd_cmd_existing(self): data = {"mode": "existing", "name": "testbr0"} self.assertEqual( cc_lxd.bridge_to_cmd(data), - (None, ["lxc", "network", "attach-profile", - "testbr0", "default", "eth0", "--force-local"])) + (None, ["network", "attach-profile", + "testbr0", "default", "eth0"])) def test_lxd_cmd_none(self): data = {"mode": "none"} @@ -183,4 +192,43 @@ class TestLxd(t_help.CiTestCase): cc_lxd.bridge_to_cmd(data), (None, None)) + +class TestLxdMaybeCleanupDefault(t_help.CiTestCase): + """Test the implementation of maybe_cleanup_default.""" + + defnet = cc_lxd._DEFAULT_NETWORK_NAME + + @mock.patch("cloudinit.config.cc_lxd._lxc") + def test_network_other_than_default_not_deleted(self, m_lxc): + """deletion or removal should only occur if bridge is default.""" + cc_lxd.maybe_cleanup_default( + net_name="lxdbr1", did_init=True, create=True, attach=True) + m_lxc.assert_not_called() + + @mock.patch("cloudinit.config.cc_lxd._lxc") + def test_did_init_false_does_not_delete(self, m_lxc): + """deletion or removal should only occur if did_init is True.""" + cc_lxd.maybe_cleanup_default( + net_name=self.defnet, did_init=False, create=True, attach=True) + m_lxc.assert_not_called() + + @mock.patch("cloudinit.config.cc_lxd._lxc") + def test_network_deleted_if_create_true(self, m_lxc): + """deletion of network should occur if create is True.""" + cc_lxd.maybe_cleanup_default( + net_name=self.defnet, did_init=True, create=True, attach=False) + m_lxc.assert_called_once_with(["network", "delete", self.defnet]) + + @mock.patch("cloudinit.config.cc_lxd._lxc") + def test_device_removed_if_attach_true(self, m_lxc): + """deletion of network should occur if create is True.""" + nic_name = "my_nic" + profile = "my_profile" + cc_lxd.maybe_cleanup_default( + net_name=self.defnet, did_init=True, create=False, attach=True, + profile=profile, nic_name=nic_name) + m_lxc.assert_called_once_with( + ["profile", "device", "remove", profile, nic_name]) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index fe492d4b..8fea6c2a 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import os.path -import shutil -import tempfile from cloudinit.config import cc_mounts @@ -18,8 +16,7 @@ class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): def setUp(self): super(TestSanitizeDevname, self).setUp() - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) + self.new_root = self.tmp_dir() self.patchOS(self.new_root) def _touch(self, path): @@ -134,4 +131,103 @@ class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): cc_mounts.sanitize_devname( 'ephemeral0.1', lambda x: disk_path, mock.Mock())) + +class TestFstabHandling(test_helpers.FilesystemMockingTestCase): + + swap_path = '/dev/sdb1' + + def setUp(self): + super(TestFstabHandling, self).setUp() + self.new_root = self.tmp_dir() + self.patchOS(self.new_root) + + self.fstab_path = os.path.join(self.new_root, 'etc/fstab') + self._makedirs('/etc') + + self.add_patch('cloudinit.config.cc_mounts.FSTAB_PATH', + 'mock_fstab_path', + self.fstab_path, + autospec=False) + + self.add_patch('cloudinit.config.cc_mounts._is_block_device', + 'mock_is_block_device', + return_value=True) + + self.add_patch('cloudinit.config.cc_mounts.util.subp', + 'mock_util_subp') + + self.mock_cloud = mock.Mock() + self.mock_log = mock.Mock() + self.mock_cloud.device_name_to_device = self.device_name_to_device + + def _makedirs(self, directory): + directory = os.path.join(self.new_root, directory.lstrip('/')) + if not os.path.exists(directory): + os.makedirs(directory) + + def device_name_to_device(self, path): + if path == 'swap': + return self.swap_path + else: + dev = None + + return dev + + def test_fstab_no_swap_device(self): + '''Ensure that cloud-init adds a discovered swap partition + to /etc/fstab.''' + + fstab_original_content = '' + fstab_expected_content = ( + '%s\tnone\tswap\tsw,comment=cloudconfig\t' + '0\t0\n' % (self.swap_path,) + ) + + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab_original_content) + + cc_mounts.handle(None, {}, self.mock_cloud, self.mock_log, []) + + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + self.assertEqual(fstab_expected_content, fstab_new_content) + + def test_fstab_same_swap_device_already_configured(self): + '''Ensure that cloud-init will not add a swap device if the same + device already exists in /etc/fstab.''' + + fstab_original_content = '%s swap swap defaults 0 0\n' % ( + self.swap_path,) + fstab_expected_content = fstab_original_content + + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab_original_content) + + cc_mounts.handle(None, {}, self.mock_cloud, self.mock_log, []) + + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + self.assertEqual(fstab_expected_content, fstab_new_content) + + def test_fstab_alternate_swap_device_already_configured(self): + '''Ensure that cloud-init will add a discovered swap device to + /etc/fstab even when there exists a swap definition on another + device.''' + + fstab_original_content = '/dev/sdc1 swap swap defaults 0 0\n' + fstab_expected_content = ( + fstab_original_content + + '%s\tnone\tswap\tsw,comment=cloudconfig\t' + '0\t0\n' % (self.swap_path,) + ) + + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab_original_content) + + cc_mounts.handle(None, {}, self.mock_cloud, self.mock_log, []) + + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + self.assertEqual(fstab_expected_content, fstab_new_content) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 695897c0..0f22e579 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,21 +3,23 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) + from cloudinit.tests.helpers import ( - FilesystemMockingTestCase, mock, skipUnlessJsonSchema) + CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema) +import copy import os from os.path import dirname import shutil -NTP_TEMPLATE = b"""\ +NTP_TEMPLATE = """\ ## template: jinja servers {{servers}} pools {{pools}} """ -TIMESYNCD_TEMPLATE = b"""\ +TIMESYNCD_TEMPLATE = """\ ## template:jinja [Time] {% if servers or pools -%} @@ -32,56 +34,88 @@ class TestNtp(FilesystemMockingTestCase): def setUp(self): super(TestNtp, self).setUp() - self.subp = util.subp self.new_root = self.tmp_dir() + self.add_patch('cloudinit.util.system_is_snappy', 'm_snappy') + self.m_snappy.return_value = False + self.add_patch('cloudinit.util.system_info', 'm_sysinfo') + self.m_sysinfo.return_value = {'dist': ('Distro', '99.1', 'Codename')} - def _get_cloud(self, distro): - self.patchUtils(self.new_root) + def _get_cloud(self, distro, sys_cfg=None): + self.new_root = self.reRoot(root=self.new_root) paths = helpers.Paths({'templates_dir': self.new_root}) cls = distros.fetch(distro) - mydist = cls(distro, {}, paths) - myds = DataSourceNone.DataSourceNone({}, mydist, paths) - return cloud.Cloud(myds, paths, {}, mydist, None) + if not sys_cfg: + sys_cfg = {} + mydist = cls(distro, sys_cfg, paths) + myds = DataSourceNone.DataSourceNone(sys_cfg, mydist, paths) + return cloud.Cloud(myds, paths, sys_cfg, mydist, None) + + def _get_template_path(self, template_name, distro, basepath=None): + # ntp.conf.{distro} -> ntp.conf.debian.tmpl + template_fn = '{0}.tmpl'.format( + template_name.replace('{distro}', distro)) + if not basepath: + basepath = self.new_root + path = os.path.join(basepath, template_fn) + return path + + def _generate_template(self, template=None): + if not template: + template = NTP_TEMPLATE + confpath = os.path.join(self.new_root, 'client.conf') + template_fn = os.path.join(self.new_root, 'client.conf.tmpl') + util.write_file(template_fn, content=template) + return (confpath, template_fn) + + def _mock_ntp_client_config(self, client=None, distro=None): + if not client: + client = 'ntp' + if not distro: + distro = 'ubuntu' + dcfg = cc_ntp.distro_ntp_client_configs(distro) + if client == 'systemd-timesyncd': + template = TIMESYNCD_TEMPLATE + else: + template = NTP_TEMPLATE + (confpath, _template_fn) = self._generate_template(template=template) + ntpconfig = copy.deepcopy(dcfg[client]) + ntpconfig['confpath'] = confpath + ntpconfig['template_name'] = os.path.basename(confpath) + return ntpconfig @mock.patch("cloudinit.config.cc_ntp.util") def test_ntp_install(self, mock_util): - """ntp_install installs via install_func when check_exe is absent.""" + """ntp_install_client runs install_func when check_exe is absent.""" mock_util.which.return_value = None # check_exe not found. install_func = mock.MagicMock() - cc_ntp.install_ntp(install_func, packages=['ntpx'], check_exe='ntpdx') - + cc_ntp.install_ntp_client(install_func, + packages=['ntpx'], check_exe='ntpdx') mock_util.which.assert_called_with('ntpdx') install_func.assert_called_once_with(['ntpx']) @mock.patch("cloudinit.config.cc_ntp.util") def test_ntp_install_not_needed(self, mock_util): - """ntp_install doesn't attempt install when check_exe is found.""" - mock_util.which.return_value = ["/usr/sbin/ntpd"] # check_exe found. + """ntp_install_client doesn't install when check_exe is found.""" + client = 'chrony' + mock_util.which.return_value = [client] # check_exe found. install_func = mock.MagicMock() - cc_ntp.install_ntp(install_func, packages=['ntp'], check_exe='ntpd') + cc_ntp.install_ntp_client(install_func, packages=[client], + check_exe=client) install_func.assert_not_called() @mock.patch("cloudinit.config.cc_ntp.util") def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util): - """ntp_install calls install_func with empty list""" + """ntp_install_client runs install_func with empty list""" mock_util.which.return_value = None # check_exe not found install_func = mock.MagicMock() - cc_ntp.install_ntp(install_func, packages=[], check_exe='timesyncd') + cc_ntp.install_ntp_client(install_func, packages=[], + check_exe='timesyncd') install_func.assert_called_once_with([]) - def test_ntp_rename_ntp_conf(self): - """When NTP_CONF exists, rename_ntp moves it.""" - ntpconf = self.tmp_path("ntp.conf", self.new_root) - util.write_file(ntpconf, "") - with mock.patch("cloudinit.config.cc_ntp.NTP_CONF", ntpconf): - cc_ntp.rename_ntp_conf() - self.assertFalse(os.path.exists(ntpconf)) - self.assertTrue(os.path.exists("{0}.dist".format(ntpconf))) - @mock.patch("cloudinit.config.cc_ntp.util") def test_reload_ntp_defaults(self, mock_util): """Test service is restarted/reloaded (defaults)""" - service = 'ntp' + service = 'ntp_service_name' cmd = ['service', service, 'restart'] cc_ntp.reload_ntp(service) mock_util.subp.assert_called_with(cmd, capture=True) @@ -89,193 +123,169 @@ class TestNtp(FilesystemMockingTestCase): @mock.patch("cloudinit.config.cc_ntp.util") def test_reload_ntp_systemd(self, mock_util): """Test service is restarted/reloaded (systemd)""" - service = 'ntp' - cmd = ['systemctl', 'reload-or-restart', service] + service = 'ntp_service_name' cc_ntp.reload_ntp(service, systemd=True) - mock_util.subp.assert_called_with(cmd, capture=True) - - @mock.patch("cloudinit.config.cc_ntp.util") - def test_reload_ntp_systemd_timesycnd(self, mock_util): - """Test service is restarted/reloaded (systemd/timesyncd)""" - service = 'systemd-timesycnd' cmd = ['systemctl', 'reload-or-restart', service] - cc_ntp.reload_ntp(service, systemd=True) mock_util.subp.assert_called_with(cmd, capture=True) + def test_ntp_rename_ntp_conf(self): + """When NTP_CONF exists, rename_ntp moves it.""" + ntpconf = self.tmp_path("ntp.conf", self.new_root) + util.write_file(ntpconf, "") + cc_ntp.rename_ntp_conf(confpath=ntpconf) + self.assertFalse(os.path.exists(ntpconf)) + self.assertTrue(os.path.exists("{0}.dist".format(ntpconf))) + def test_ntp_rename_ntp_conf_skip_missing(self): """When NTP_CONF doesn't exist rename_ntp doesn't create a file.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) self.assertFalse(os.path.exists(ntpconf)) - with mock.patch("cloudinit.config.cc_ntp.NTP_CONF", ntpconf): - cc_ntp.rename_ntp_conf() + cc_ntp.rename_ntp_conf(confpath=ntpconf) self.assertFalse(os.path.exists("{0}.dist".format(ntpconf))) self.assertFalse(os.path.exists(ntpconf)) - def test_write_ntp_config_template_from_ntp_conf_tmpl_with_servers(self): - """write_ntp_config_template reads content from ntp.conf.tmpl. - - It reads ntp.conf.tmpl if present and renders the value from servers - key. When no pools key is defined, template is rendered using an empty - list for pools. - """ - distro = 'ubuntu' - cfg = { - 'servers': ['192.168.2.1', '192.168.2.2'] - } - mycloud = self._get_cloud(distro) - ntp_conf = self.tmp_path("ntp.conf", self.new_root) # Doesn't exist - # Create ntp.conf.tmpl - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) - content = util.read_file_or_url('file://' + ntp_conf).contents + def test_write_ntp_config_template_uses_ntp_conf_distro_no_servers(self): + """write_ntp_config_template reads from $client.conf.distro.tmpl""" + servers = [] + pools = ['10.0.0.1', '10.0.0.2'] + (confpath, template_fn) = self._generate_template() + mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR' + with mock.patch(mock_path, self.new_root): + cc_ntp.write_ntp_config_template('ubuntu', + servers=servers, pools=pools, + path=confpath, + template_fn=template_fn, + template=None) self.assertEqual( - "servers ['192.168.2.1', '192.168.2.2']\npools []\n", - content.decode()) + "servers []\npools ['10.0.0.1', '10.0.0.2']\n", + util.load_file(confpath)) - def test_write_ntp_config_template_uses_ntp_conf_distro_no_servers(self): - """write_ntp_config_template reads content from ntp.conf.distro.tmpl. + def test_write_ntp_config_template_defaults_pools_w_empty_lists(self): + """write_ntp_config_template defaults pools servers upon empty config. - It reads ntp.conf.<distro>.tmpl before attempting ntp.conf.tmpl. It - renders the value from the keys servers and pools. When no - servers value is present, template is rendered using an empty list. + When both pools and servers are empty, default NR_POOL_SERVERS get + configured. """ distro = 'ubuntu' - cfg = { - 'pools': ['10.0.0.1', '10.0.0.2'] - } - mycloud = self._get_cloud(distro) - ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist - # Create ntp.conf.tmpl which isn't read - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(b'NOT READ: ntp.conf.<distro>.tmpl is primary') - # Create ntp.conf.tmpl.<distro> - with open('{0}.{1}.tmpl'.format(ntp_conf, distro), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) - content = util.read_file_or_url('file://' + ntp_conf).contents + pools = cc_ntp.generate_server_names(distro) + servers = [] + (confpath, template_fn) = self._generate_template() + mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR' + with mock.patch(mock_path, self.new_root): + cc_ntp.write_ntp_config_template(distro, + servers=servers, pools=pools, + path=confpath, + template_fn=template_fn, + template=None) self.assertEqual( - "servers []\npools ['10.0.0.1', '10.0.0.2']\n", - content.decode()) + "servers []\npools {0}\n".format(pools), + util.load_file(confpath)) - def test_write_ntp_config_template_defaults_pools_when_empty_lists(self): - """write_ntp_config_template defaults pools servers upon empty config. + def test_defaults_pools_empty_lists_sles(self): + """write_ntp_config_template defaults opensuse pools upon empty config. When both pools and servers are empty, default NR_POOL_SERVERS get configured. """ - distro = 'ubuntu' - mycloud = self._get_cloud(distro) - ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist - # Create ntp.conf.tmpl - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf) - content = util.read_file_or_url('file://' + ntp_conf).contents - default_pools = [ - "{0}.{1}.pool.ntp.org".format(x, distro) - for x in range(0, cc_ntp.NR_POOL_SERVERS)] + distro = 'sles' + default_pools = cc_ntp.generate_server_names(distro) + (confpath, template_fn) = self._generate_template() + + cc_ntp.write_ntp_config_template(distro, + servers=[], pools=[], + path=confpath, + template_fn=template_fn, + template=None) + for pool in default_pools: + self.assertIn('opensuse', pool) self.assertEqual( "servers []\npools {0}\n".format(default_pools), - content.decode()) + util.load_file(confpath)) self.assertIn( "Adding distro default ntp pool servers: {0}".format( ",".join(default_pools)), self.logs.getvalue()) - @mock.patch("cloudinit.config.cc_ntp.ntp_installable") - def test_ntp_handler_mocked_template(self, m_ntp_install): - """Test ntp handler renders ubuntu ntp.conf template.""" - pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] - servers = ['192.168.23.3', '192.168.23.4'] - cfg = { - 'ntp': { - 'pools': pools, - 'servers': servers - } - } - mycloud = self._get_cloud('ubuntu') - ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist - m_ntp_install.return_value = True - - # Create ntp.conf.tmpl - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - with mock.patch.object(util, 'which', return_value=None): - cc_ntp.handle('notimportant', cfg, mycloud, None, None) - - content = util.read_file_or_url('file://' + ntp_conf).contents - self.assertEqual( - 'servers {0}\npools {1}\n'.format(servers, pools), - content.decode()) - - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_handler_mocked_template_snappy(self, m_util): - """Test ntp handler renders timesycnd.conf template on snappy.""" + def test_timesyncd_template(self): + """Test timesycnd template is correct""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] servers = ['192.168.23.3', '192.168.23.4'] - cfg = { - 'ntp': { - 'pools': pools, - 'servers': servers - } - } - mycloud = self._get_cloud('ubuntu') - m_util.system_is_snappy.return_value = True - - # Create timesyncd.conf.tmpl - tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) - template = '{0}.tmpl'.format(tsyncd_conf) - with open(template, 'wb') as stream: - stream.write(TIMESYNCD_TEMPLATE) - - with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): - cc_ntp.handle('notimportant', cfg, mycloud, None, None) - - content = util.read_file_or_url('file://' + tsyncd_conf).contents + (confpath, template_fn) = self._generate_template( + template=TIMESYNCD_TEMPLATE) + cc_ntp.write_ntp_config_template('ubuntu', + servers=servers, pools=pools, + path=confpath, + template_fn=template_fn, + template=None) self.assertEqual( "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)), - content.decode()) - - def test_ntp_handler_real_distro_templates(self): - """Test ntp handler renders the shipped distro ntp.conf templates.""" + util.load_file(confpath)) + + def test_distro_ntp_client_configs(self): + """Test we have updated ntp client configs on different distros""" + delta = copy.deepcopy(cc_ntp.DISTRO_CLIENT_CONFIG) + base = copy.deepcopy(cc_ntp.NTP_CLIENT_CONFIG) + # confirm no-delta distros match the base config + for distro in cc_ntp.distros: + if distro not in delta: + result = cc_ntp.distro_ntp_client_configs(distro) + self.assertEqual(base, result) + # for distros with delta, ensure the merged config values match + # what is set in the delta + for distro in delta.keys(): + result = cc_ntp.distro_ntp_client_configs(distro) + for client in delta[distro].keys(): + for key in delta[distro][client].keys(): + self.assertEqual(delta[distro][client][key], + result[client][key]) + + def test_ntp_handler_real_distro_ntp_templates(self): + """Test ntp handler renders the shipped distro ntp client templates.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] servers = ['192.168.23.3', '192.168.23.4'] - cfg = { - 'ntp': { - 'pools': pools, - 'servers': servers - } - } - ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist - for distro in ('debian', 'ubuntu', 'fedora', 'rhel', 'sles'): - mycloud = self._get_cloud(distro) - root_dir = dirname(dirname(os.path.realpath(util.__file__))) - tmpl_file = os.path.join( - '{0}/templates/ntp.conf.{1}.tmpl'.format(root_dir, distro)) - # Create a copy in our tmp_dir - shutil.copy( - tmpl_file, - os.path.join(self.new_root, 'ntp.conf.%s.tmpl' % distro)) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - with mock.patch.object(util, 'which', return_value=[True]): - cc_ntp.handle('notimportant', cfg, mycloud, None, None) - - content = util.read_file_or_url('file://' + ntp_conf).contents - expected_servers = '\n'.join([ - 'server {0} iburst'.format(server) for server in servers]) - self.assertIn( - expected_servers, content.decode(), - 'failed to render ntp.conf for distro:{0}'.format(distro)) - expected_pools = '\n'.join([ - 'pool {0} iburst'.format(pool) for pool in pools]) - self.assertIn( - expected_pools, content.decode(), - 'failed to render ntp.conf for distro:{0}'.format(distro)) + for client in ['ntp', 'systemd-timesyncd', 'chrony']: + for distro in cc_ntp.distros: + distro_cfg = cc_ntp.distro_ntp_client_configs(distro) + ntpclient = distro_cfg[client] + confpath = ( + os.path.join(self.new_root, ntpclient.get('confpath')[1:])) + template = ntpclient.get('template_name') + # find sourcetree template file + root_dir = ( + dirname(dirname(os.path.realpath(util.__file__))) + + '/templates') + source_fn = self._get_template_path(template, distro, + basepath=root_dir) + template_fn = self._get_template_path(template, distro) + # don't fail if cloud-init doesn't have a template for + # a distro,client pair + if not os.path.exists(source_fn): + continue + # Create a copy in our tmp_dir + shutil.copy(source_fn, template_fn) + cc_ntp.write_ntp_config_template(distro, servers=servers, + pools=pools, path=confpath, + template_fn=template_fn) + content = util.load_file(confpath) + if client in ['ntp', 'chrony']: + expected_servers = '\n'.join([ + 'server {0} iburst'.format(srv) for srv in servers]) + print('distro=%s client=%s' % (distro, client)) + self.assertIn(expected_servers, content, + ('failed to render {0} conf' + ' for distro:{1}'.format(client, distro))) + expected_pools = '\n'.join([ + 'pool {0} iburst'.format(pool) for pool in pools]) + self.assertIn(expected_pools, content, + ('failed to render {0} conf' + ' for distro:{1}'.format(client, distro))) + elif client == 'systemd-timesyncd': + expected_content = ( + "# cloud-init generated file\n" + + "# See timesyncd.conf(5) for details.\n\n" + + "[Time]\nNTP=%s %s \n" % (" ".join(servers), + " ".join(pools))) + self.assertEqual(expected_content, content) def test_no_ntpcfg_does_nothing(self): """When no ntp section is defined handler logs a warning and noops.""" @@ -285,95 +295,96 @@ class TestNtp(FilesystemMockingTestCase): 'not present or disabled by cfg\n', self.logs.getvalue()) - def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_schema_validation_allows_empty_ntp_config(self, + m_select): """Ntp schema validation allows for an empty ntp: configuration.""" valid_empty_configs = [{'ntp': {}}, {'ntp': None}] - distro = 'ubuntu' - cc = self._get_cloud(distro) - ntp_conf = os.path.join(self.new_root, 'ntp.conf') - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) for valid_empty_config in valid_empty_configs: - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.handle('cc_ntp', valid_empty_config, cc, None, []) - with open(ntp_conf) as stream: - content = stream.read() - default_pools = [ - "{0}.{1}.pool.ntp.org".format(x, distro) - for x in range(0, cc_ntp.NR_POOL_SERVERS)] - self.assertEqual( - "servers []\npools {0}\n".format(default_pools), - content) - self.assertNotIn('Invalid config:', self.logs.getvalue()) + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', valid_empty_config, mycloud, None, []) + pools = cc_ntp.generate_server_names(mycloud.distro.name) + self.assertEqual( + "servers []\npools {0}\n".format(pools), + util.load_file(confpath)) + self.assertNotIn('Invalid config:', self.logs.getvalue()) @skipUnlessJsonSchema() - def test_ntp_handler_schema_validation_warns_non_string_item_type(self): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_schema_validation_warns_non_string_item_type(self, + m_sel): """Ntp schema validation warns of non-strings in pools or servers. Schema validation is not strict, so ntp config is still be rendered. """ invalid_config = {'ntp': {'pools': [123], 'servers': ['valid', None]}} - cc = self._get_cloud('ubuntu') - ntp_conf = os.path.join(self.new_root, 'ntp.conf') - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) - self.assertIn( - "Invalid config:\nntp.pools.0: 123 is not of type 'string'\n" - "ntp.servers.1: None is not of type 'string'", - self.logs.getvalue()) - with open(ntp_conf) as stream: - content = stream.read() - self.assertEqual("servers ['valid', None]\npools [123]\n", content) + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_sel.return_value = ntpconfig + cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) + self.assertIn( + "Invalid config:\nntp.pools.0: 123 is not of type 'string'\n" + "ntp.servers.1: None is not of type 'string'", + self.logs.getvalue()) + self.assertEqual("servers ['valid', None]\npools [123]\n", + util.load_file(confpath)) @skipUnlessJsonSchema() - def test_ntp_handler_schema_validation_warns_of_non_array_type(self): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_schema_validation_warns_of_non_array_type(self, + m_select): """Ntp schema validation warns of non-array pools or servers types. Schema validation is not strict, so ntp config is still be rendered. """ invalid_config = {'ntp': {'pools': 123, 'servers': 'non-array'}} - cc = self._get_cloud('ubuntu') - ntp_conf = os.path.join(self.new_root, 'ntp.conf') - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) - self.assertIn( - "Invalid config:\nntp.pools: 123 is not of type 'array'\n" - "ntp.servers: 'non-array' is not of type 'array'", - self.logs.getvalue()) - with open(ntp_conf) as stream: - content = stream.read() - self.assertEqual("servers non-array\npools 123\n", content) + + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) + self.assertIn( + "Invalid config:\nntp.pools: 123 is not of type 'array'\n" + "ntp.servers: 'non-array' is not of type 'array'", + self.logs.getvalue()) + self.assertEqual("servers non-array\npools 123\n", + util.load_file(confpath)) @skipUnlessJsonSchema() - def test_ntp_handler_schema_validation_warns_invalid_key_present(self): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_schema_validation_warns_invalid_key_present(self, + m_select): """Ntp schema validation warns of invalid keys present in ntp config. Schema validation is not strict, so ntp config is still be rendered. """ invalid_config = { 'ntp': {'invalidkey': 1, 'pools': ['0.mycompany.pool.ntp.org']}} - cc = self._get_cloud('ubuntu') - ntp_conf = os.path.join(self.new_root, 'ntp.conf') - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) - self.assertIn( - "Invalid config:\nntp: Additional properties are not allowed " - "('invalidkey' was unexpected)", - self.logs.getvalue()) - with open(ntp_conf) as stream: - content = stream.read() - self.assertEqual( - "servers []\npools ['0.mycompany.pool.ntp.org']\n", - content) + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) + self.assertIn( + "Invalid config:\nntp: Additional properties are not allowed " + "('invalidkey' was unexpected)", + self.logs.getvalue()) + self.assertEqual( + "servers []\npools ['0.mycompany.pool.ntp.org']\n", + util.load_file(confpath)) @skipUnlessJsonSchema() - def test_ntp_handler_schema_validation_warns_of_duplicates(self): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_schema_validation_warns_of_duplicates(self, m_select): """Ntp schema validation warns of duplicates in servers or pools. Schema validation is not strict, so ntp config is still be rendered. @@ -381,74 +392,330 @@ class TestNtp(FilesystemMockingTestCase): invalid_config = { 'ntp': {'pools': ['0.mypool.org', '0.mypool.org'], 'servers': ['10.0.0.1', '10.0.0.1']}} - cc = self._get_cloud('ubuntu') - ntp_conf = os.path.join(self.new_root, 'ntp.conf') - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) - self.assertIn( - "Invalid config:\nntp.pools: ['0.mypool.org', '0.mypool.org'] has " - "non-unique elements\nntp.servers: ['10.0.0.1', '10.0.0.1'] has " - "non-unique elements", - self.logs.getvalue()) - with open(ntp_conf) as stream: - content = stream.read() - self.assertEqual( - "servers ['10.0.0.1', '10.0.0.1']\n" - "pools ['0.mypool.org', '0.mypool.org']\n", - content) + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) + self.assertIn( + "Invalid config:\nntp.pools: ['0.mypool.org', '0.mypool.org']" + " has non-unique elements\nntp.servers: " + "['10.0.0.1', '10.0.0.1'] has non-unique elements", + self.logs.getvalue()) + self.assertEqual( + "servers ['10.0.0.1', '10.0.0.1']\n" + "pools ['0.mypool.org', '0.mypool.org']\n", + util.load_file(confpath)) - @mock.patch("cloudinit.config.cc_ntp.ntp_installable") - def test_ntp_handler_timesyncd(self, m_ntp_install): + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_timesyncd(self, m_select): """Test ntp handler configures timesyncd""" - m_ntp_install.return_value = False - distro = 'ubuntu' - cfg = { - 'servers': ['192.168.2.1', '192.168.2.2'], - 'pools': ['0.mypool.org'], - } - mycloud = self._get_cloud(distro) - tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) - # Create timesyncd.conf.tmpl - template = '{0}.tmpl'.format(tsyncd_conf) - print(template) - with open(template, 'wb') as stream: - stream.write(TIMESYNCD_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud, tsyncd_conf, - template='timesyncd.conf') - - content = util.read_file_or_url('file://' + tsyncd_conf).contents - self.assertEqual( - "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n", - content.decode()) + servers = ['192.168.2.1', '192.168.2.2'] + pools = ['0.mypool.org'] + cfg = {'ntp': {'servers': servers, 'pools': pools}} + client = 'systemd-timesyncd' + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro, + client=client) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', cfg, mycloud, None, []) + self.assertEqual( + "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n", + util.load_file(confpath)) + + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_handler_enabled_false(self, m_select): + """Test ntp handler does not run if enabled: false """ + cfg = {'ntp': {'enabled': False}} + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + self.assertEqual(0, m_select.call_count) + + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + @mock.patch("cloudinit.distros.Distro.uses_systemd") + def test_ntp_the_whole_package(self, m_sysd, m_select): + """Test enabled config renders template, and restarts service """ + cfg = {'ntp': {'enabled': True}} + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + service_name = ntpconfig['service_name'] + m_select.return_value = ntpconfig + pools = cc_ntp.generate_server_names(mycloud.distro.name) + # force uses systemd path + m_sysd.return_value = True + with mock.patch('cloudinit.config.cc_ntp.util') as m_util: + # allow use of util.mergemanydict + m_util.mergemanydict.side_effect = util.mergemanydict + # default client is present + m_util.which.return_value = True + # use the config 'enabled' value + m_util.is_false.return_value = util.is_false( + cfg['ntp']['enabled']) + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + m_util.subp.assert_called_with( + ['systemctl', 'reload-or-restart', + service_name], capture=True) + self.assertEqual( + "servers []\npools {0}\n".format(pools), + util.load_file(confpath)) + + def test_opensuse_picks_chrony(self): + """Test opensuse picks chrony or ntp on certain distro versions""" + # < 15.0 => ntp + self.m_sysinfo.return_value = {'dist': + ('openSUSE', '13.2', 'Harlequin')} + mycloud = self._get_cloud('opensuse') + expected_client = mycloud.distro.preferred_ntp_clients[0] + self.assertEqual('ntp', expected_client) + + # >= 15.0 and not openSUSE => chrony + self.m_sysinfo.return_value = {'dist': + ('SLES', '15.0', + 'SUSE Linux Enterprise Server 15')} + mycloud = self._get_cloud('sles') + expected_client = mycloud.distro.preferred_ntp_clients[0] + self.assertEqual('chrony', expected_client) + + # >= 15.0 and openSUSE and ver != 42 => chrony + self.m_sysinfo.return_value = {'dist': ('openSUSE Tumbleweed', + '20180326', + 'timbleweed')} + mycloud = self._get_cloud('opensuse') + expected_client = mycloud.distro.preferred_ntp_clients[0] + self.assertEqual('chrony', expected_client) + + def test_ubuntu_xenial_picks_ntp(self): + """Test Ubuntu picks ntp on xenial release""" + + self.m_sysinfo.return_value = {'dist': ('Ubuntu', '16.04', 'xenial')} + mycloud = self._get_cloud('ubuntu') + expected_client = mycloud.distro.preferred_ntp_clients[0] + self.assertEqual('ntp', expected_client) - def test_write_ntp_config_template_defaults_pools_empty_lists_sles(self): - """write_ntp_config_template defaults pools servers upon empty config. + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_snappy_system_picks_timesyncd(self, m_which): + """Test snappy systems prefer installed clients""" - When both pools and servers are empty, default NR_POOL_SERVERS get - configured. - """ - distro = 'sles' - mycloud = self._get_cloud(distro) - ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist - # Create ntp.conf.tmpl - with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: - stream.write(NTP_TEMPLATE) - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf) - content = util.read_file_or_url('file://' + ntp_conf).contents - default_pools = [ - "{0}.opensuse.pool.ntp.org".format(x) - for x in range(0, cc_ntp.NR_POOL_SERVERS)] - self.assertEqual( - "servers []\npools {0}\n".format(default_pools), - content.decode()) - self.assertIn( - "Adding distro default ntp pool servers: {0}".format( - ",".join(default_pools)), - self.logs.getvalue()) + # we are on ubuntu-core here + self.m_snappy.return_value = True + # ubuntu core systems will have timesyncd installed + m_which.side_effect = iter([None, '/lib/systemd/systemd-timesyncd', + None, None, None]) + distro = 'ubuntu' + mycloud = self._get_cloud(distro) + distro_configs = cc_ntp.distro_ntp_client_configs(distro) + expected_client = 'systemd-timesyncd' + expected_cfg = distro_configs[expected_client] + expected_calls = [] + # we only get to timesyncd + for client in mycloud.distro.preferred_ntp_clients[0:2]: + cfg = distro_configs[client] + expected_calls.append(mock.call(cfg['check_exe'])) + result = cc_ntp.select_ntp_client(None, mycloud.distro) + m_which.assert_has_calls(expected_calls) + self.assertEqual(sorted(expected_cfg), sorted(cfg)) + self.assertEqual(sorted(expected_cfg), sorted(result)) + + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_ntp_distro_searches_all_preferred_clients(self, m_which): + """Test select_ntp_client search all distro perferred clients """ + # nothing is installed + m_which.return_value = None + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + distro_configs = cc_ntp.distro_ntp_client_configs(distro) + expected_client = mycloud.distro.preferred_ntp_clients[0] + expected_cfg = distro_configs[expected_client] + expected_calls = [] + for client in mycloud.distro.preferred_ntp_clients: + cfg = distro_configs[client] + expected_calls.append(mock.call(cfg['check_exe'])) + cc_ntp.select_ntp_client({}, mycloud.distro) + m_which.assert_has_calls(expected_calls) + self.assertEqual(sorted(expected_cfg), sorted(cfg)) + + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_user_cfg_ntp_client_auto_uses_distro_clients(self, m_which): + """Test user_cfg.ntp_client='auto' defaults to distro search""" + # nothing is installed + m_which.return_value = None + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + distro_configs = cc_ntp.distro_ntp_client_configs(distro) + expected_client = mycloud.distro.preferred_ntp_clients[0] + expected_cfg = distro_configs[expected_client] + expected_calls = [] + for client in mycloud.distro.preferred_ntp_clients: + cfg = distro_configs[client] + expected_calls.append(mock.call(cfg['check_exe'])) + cc_ntp.select_ntp_client('auto', mycloud.distro) + m_which.assert_has_calls(expected_calls) + self.assertEqual(sorted(expected_cfg), sorted(cfg)) + + @mock.patch('cloudinit.config.cc_ntp.write_ntp_config_template') + @mock.patch('cloudinit.cloud.Cloud.get_template_filename') + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_ntp_custom_client_overrides_installed_clients(self, m_which, + m_tmpfn, m_write): + """Test user client is installed despite other clients present """ + client = 'ntpdate' + cfg = {'ntp': {'ntp_client': client}} + for distro in cc_ntp.distros: + # client is not installed + m_which.side_effect = iter([None]) + mycloud = self._get_cloud(distro) + with mock.patch.object(mycloud.distro, + 'install_packages') as m_install: + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + m_install.assert_called_with([client]) + m_which.assert_called_with(client) + + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_ntp_system_config_overrides_distro_builtin_clients(self, m_which): + """Test distro system_config overrides builtin preferred ntp clients""" + system_client = 'chrony' + sys_cfg = {'ntp_client': system_client} + # no clients installed + m_which.return_value = None + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro, sys_cfg=sys_cfg) + distro_configs = cc_ntp.distro_ntp_client_configs(distro) + expected_cfg = distro_configs[system_client] + result = cc_ntp.select_ntp_client(None, mycloud.distro) + self.assertEqual(sorted(expected_cfg), sorted(result)) + m_which.assert_has_calls([]) + + @mock.patch('cloudinit.config.cc_ntp.util.which') + def test_ntp_user_config_overrides_system_cfg(self, m_which): + """Test user-data overrides system_config ntp_client""" + system_client = 'chrony' + sys_cfg = {'ntp_client': system_client} + user_client = 'systemd-timesyncd' + # no clients installed + m_which.return_value = None + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro, sys_cfg=sys_cfg) + distro_configs = cc_ntp.distro_ntp_client_configs(distro) + expected_cfg = distro_configs[user_client] + result = cc_ntp.select_ntp_client(user_client, mycloud.distro) + self.assertEqual(sorted(expected_cfg), sorted(result)) + m_which.assert_has_calls([]) + + @mock.patch('cloudinit.config.cc_ntp.reload_ntp') + @mock.patch('cloudinit.config.cc_ntp.install_ntp_client') + def test_ntp_user_provided_config_with_template(self, m_install, m_reload): + custom = r'\n#MyCustomTemplate' + user_template = NTP_TEMPLATE + custom + confpath = os.path.join(self.new_root, 'etc/myntp/myntp.conf') + cfg = { + 'ntp': { + 'pools': ['mypool.org'], + 'ntp_client': 'myntpd', + 'config': { + 'check_exe': 'myntpd', + 'confpath': confpath, + 'packages': ['myntp'], + 'service_name': 'myntp', + 'template': user_template, + } + } + } + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR' + with mock.patch(mock_path, self.new_root): + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + self.assertEqual( + "servers []\npools ['mypool.org']\n%s" % custom, + util.load_file(confpath)) + + @mock.patch('cloudinit.config.cc_ntp.supplemental_schema_validation') + @mock.patch('cloudinit.config.cc_ntp.reload_ntp') + @mock.patch('cloudinit.config.cc_ntp.install_ntp_client') + @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') + def test_ntp_user_provided_config_template_only(self, m_select, m_install, + m_reload, m_schema): + """Test custom template for default client""" + custom = r'\n#MyCustomTemplate' + user_template = NTP_TEMPLATE + custom + client = 'chrony' + cfg = { + 'pools': ['mypool.org'], + 'ntp_client': client, + 'config': { + 'template': user_template, + } + } + expected_merged_cfg = { + 'check_exe': 'chronyd', + 'confpath': '{tmpdir}/client.conf'.format(tmpdir=self.new_root), + 'template_name': 'client.conf', 'template': user_template, + 'service_name': 'chrony', 'packages': ['chrony']} + for distro in cc_ntp.distros: + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(client=client, + distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR' + with mock.patch(mock_path, self.new_root): + cc_ntp.handle('notimportant', + {'ntp': cfg}, mycloud, None, None) + self.assertEqual( + "servers []\npools ['mypool.org']\n%s" % custom, + util.load_file(confpath)) + m_schema.assert_called_with(expected_merged_cfg) + + +class TestSupplementalSchemaValidation(CiTestCase): + + def test_error_on_missing_keys(self): + """ValueError raised reporting any missing required ntp:config keys""" + cfg = {} + match = (r'Invalid ntp configuration:\\nMissing required ntp:config' + ' keys: check_exe, confpath, packages, service_name') + with self.assertRaisesRegex(ValueError, match): + cc_ntp.supplemental_schema_validation(cfg) + + def test_error_requiring_either_template_or_template_name(self): + """ValueError raised if both template not template_name are None.""" + cfg = {'confpath': 'someconf', 'check_exe': '', 'service_name': '', + 'template': None, 'template_name': None, 'packages': []} + match = (r'Invalid ntp configuration:\\nEither ntp:config:template' + ' or ntp:config:template_name values are required') + with self.assertRaisesRegex(ValueError, match): + cc_ntp.supplemental_schema_validation(cfg) + + def test_error_on_non_list_values(self): + """ValueError raised when packages is not of type list.""" + cfg = {'confpath': 'someconf', 'check_exe': '', 'service_name': '', + 'template': 'asdf', 'template_name': None, 'packages': 'NOPE'} + match = (r'Invalid ntp configuration:\\nExpected a list of required' + ' package names for ntp:config:packages. Found \\(NOPE\\)') + with self.assertRaisesRegex(ValueError, match): + cc_ntp.supplemental_schema_validation(cfg) + + def test_error_on_non_string_values(self): + """ValueError raised for any values expected as string type.""" + cfg = {'confpath': 1, 'check_exe': 2, 'service_name': 3, + 'template': 4, 'template_name': 5, 'packages': []} + errors = [ + 'Expected a config file path ntp:config:confpath. Found (1)', + 'Expected a string type for ntp:config:check_exe. Found (2)', + 'Expected a string type for ntp:config:service_name. Found (3)', + 'Expected a string type for ntp:config:template. Found (4)', + 'Expected a string type for ntp:config:template_name. Found (5)'] + with self.assertRaises(ValueError) as context_mgr: + cc_ntp.supplemental_schema_validation(cfg) + error_msg = str(context_mgr.exception) + for error in errors: + self.assertIn(error, error_msg) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 7a7ba1ff..feca56c2 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -147,13 +147,15 @@ class TestResizefs(CiTestCase): def test_resize_ufs_cmd_return(self): mount_point = '/' devpth = '/dev/sda2' - self.assertEqual(('growfs', devpth), + self.assertEqual(('growfs', '-y', devpth), _resize_ufs(mount_point, devpth)) + @mock.patch('cloudinit.util.is_container', return_value=False) @mock.patch('cloudinit.util.get_mount_info') @mock.patch('cloudinit.util.get_device_info_from_zpool') @mock.patch('cloudinit.util.parse_mount') - def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount): + def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount, + is_container): devpth = 'vmzroot/ROOT/freebsd' disk = 'gpt/system' fs_type = 'zfs' @@ -354,8 +356,10 @@ class TestMaybeGetDevicePathAsWritableBlock(CiTestCase): ('btrfs', 'filesystem', 'resize', 'max', '/'), _resize_btrfs("/", "/dev/sda1")) + @mock.patch('cloudinit.util.is_container', return_value=True) @mock.patch('cloudinit.util.is_FreeBSD') - def test_maybe_get_writable_device_path_zfs_freebsd(self, freebsd): + def test_maybe_get_writable_device_path_zfs_freebsd(self, freebsd, + m_is_container): freebsd.return_value = True info = 'dev=gpt/system mnt_point=/ path=/' devpth = maybe_get_writable_device_path('gpt/system', info, LOG) diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index dbbb2717..9ce334ac 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -1,10 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config import cc_runcmd +from cloudinit.config.cc_runcmd import handle, schema from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) from cloudinit.tests.helpers import ( - FilesystemMockingTestCase, skipUnlessJsonSchema) + CiTestCase, FilesystemMockingTestCase, SchemaTestCaseMixin, + skipUnlessJsonSchema) import logging import os @@ -35,7 +36,7 @@ class TestRuncmd(FilesystemMockingTestCase): """When the provided config doesn't contain runcmd, skip it.""" cfg = {} mycloud = self._get_cloud('ubuntu') - cc_runcmd.handle('notimportant', cfg, mycloud, LOG, None) + handle('notimportant', cfg, mycloud, LOG, None) self.assertIn( "Skipping module named notimportant, no 'runcmd' key", self.logs.getvalue()) @@ -44,7 +45,7 @@ class TestRuncmd(FilesystemMockingTestCase): """Commands which can't be converted to shell will raise errors.""" invalid_config = {'runcmd': 1} cc = self._get_cloud('ubuntu') - cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, []) + handle('cc_runcmd', invalid_config, cc, LOG, []) self.assertIn( 'Failed to shellify 1 into file' ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd', @@ -59,7 +60,7 @@ class TestRuncmd(FilesystemMockingTestCase): """ invalid_config = {'runcmd': 1} cc = self._get_cloud('ubuntu') - cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, []) + handle('cc_runcmd', invalid_config, cc, LOG, []) self.assertIn( 'Invalid config:\nruncmd: 1 is not of type \'array\'', self.logs.getvalue()) @@ -75,7 +76,7 @@ class TestRuncmd(FilesystemMockingTestCase): invalid_config = { 'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} cc = self._get_cloud('ubuntu') - cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, []) + handle('cc_runcmd', invalid_config, cc, LOG, []) expected_warnings = [ 'runcmd.1: 20 is not valid under any of the given schemas', 'runcmd.3: {\'a\': \'n\'} is not valid under any of the given' @@ -90,7 +91,7 @@ class TestRuncmd(FilesystemMockingTestCase): """Valid runcmd schema is written to a runcmd shell script.""" valid_config = {'runcmd': [['ls', '/']]} cc = self._get_cloud('ubuntu') - cc_runcmd.handle('cc_runcmd', valid_config, cc, LOG, []) + handle('cc_runcmd', valid_config, cc, LOG, []) runcmd_file = os.path.join( self.new_root, 'var/lib/cloud/instances/iid-datasource-none/scripts/runcmd') @@ -99,4 +100,22 @@ class TestRuncmd(FilesystemMockingTestCase): self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode)) +@skipUnlessJsonSchema() +class TestSchema(CiTestCase, SchemaTestCaseMixin): + """Directly test schema rather than through handle.""" + + schema = schema + + def test_duplicates_are_fine_array_array(self): + """Duplicated commands array/array entries are allowed.""" + self.assertSchemaValid( + [["echo", "bye"], ["echo", "bye"]], + "command entries can be duplicate.") + + def test_duplicates_are_fine_array_string(self): + """Duplicated commands array/string entries are allowed.""" + self.assertSchemaValid( + ["echo bye", "echo bye"], + "command entries can be duplicate.") + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index ac41f124..1bad07f6 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -4,7 +4,7 @@ from cloudinit.config.schema import ( CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file, get_schema_doc, get_schema, validate_cloudconfig_file, validate_cloudconfig_schema, main) -from cloudinit.util import subp, write_file +from cloudinit.util import write_file from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema @@ -134,22 +134,35 @@ class ValidateCloudConfigFileTest(CiTestCase): with self.assertRaises(SchemaValidationError) as context_mgr: validate_cloudconfig_file(self.config_file, {}) self.assertEqual( - 'Cloud config schema errors: header: File {0} needs to begin with ' - '"{1}"'.format(self.config_file, CLOUD_CONFIG_HEADER.decode()), + 'Cloud config schema errors: format-l1.c1: File {0} needs to begin' + ' with "{1}"'.format( + self.config_file, CLOUD_CONFIG_HEADER.decode()), str(context_mgr.exception)) - def test_validateconfig_file_error_on_non_yaml_format(self): - """On non-yaml format, validate_cloudconfig_file errors.""" + def test_validateconfig_file_error_on_non_yaml_scanner_error(self): + """On non-yaml scan issues, validate_cloudconfig_file errors.""" + # Generate a scanner error by providing text on a single line with + # improper indent. + write_file(self.config_file, '#cloud-config\nasdf:\nasdf') + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_file(self.config_file, {}) + self.assertIn( + 'schema errors: format-l3.c1: File {0} is not valid yaml.'.format( + self.config_file), + str(context_mgr.exception)) + + def test_validateconfig_file_error_on_non_yaml_parser_error(self): + """On non-yaml parser issues, validate_cloudconfig_file errors.""" write_file(self.config_file, '#cloud-config\n{}}') with self.assertRaises(SchemaValidationError) as context_mgr: validate_cloudconfig_file(self.config_file, {}) self.assertIn( - 'schema errors: format: File {0} is not valid yaml.'.format( + 'schema errors: format-l2.c3: File {0} is not valid yaml.'.format( self.config_file), str(context_mgr.exception)) @skipUnlessJsonSchema() - def test_validateconfig_file_sctricty_validates_schema(self): + def test_validateconfig_file_sctrictly_validates_schema(self): """validate_cloudconfig_file raises errors on invalid schema.""" schema = { 'properties': {'p1': {'type': 'string', 'format': 'hostname'}}} @@ -342,6 +355,20 @@ class MainTest(CiTestCase): 'Expected either --config-file argument or --doc\n', m_stderr.getvalue()) + def test_main_absent_config_file(self): + """Main exits non-zero when config file is absent.""" + myargs = ['mycmd', '--annotate', '--config-file', 'NOT_A_FILE'] + with mock.patch('sys.exit', side_effect=self.sys_exit): + with mock.patch('sys.argv', myargs): + with mock.patch('sys.stderr', new_callable=StringIO) as \ + m_stderr: + with self.assertRaises(SystemExit) as context_manager: + main() + self.assertEqual(1, context_manager.exception.code) + self.assertEqual( + 'Configfile NOT_A_FILE does not exist\n', + m_stderr.getvalue()) + def test_main_prints_docs(self): """When --doc parameter is provided, main generates documentation.""" myargs = ['mycmd', '--doc'] @@ -379,8 +406,14 @@ class CloudTestsIntegrationTest(CiTestCase): integration_testdir = os.path.sep.join( [testsdir, 'cloud_tests', 'testcases']) errors = [] - out, _ = subp(['find', integration_testdir, '-name', '*yaml']) - for filename in out.splitlines(): + + yaml_files = [] + for root, _dirnames, filenames in os.walk(integration_testdir): + yaml_files.extend([os.path.join(root, f) + for f in filenames if f.endswith(".yaml")]) + self.assertTrue(len(yaml_files) > 0) + + for filename in yaml_files: test_cfg = safe_load(open(filename)) cloud_config = test_cfg.get('cloud_config') if cloud_config: diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index f51358da..3a5072c7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -100,7 +100,7 @@ def make_dict(max_depth, seed=None): class TestSimpleRun(helpers.ResourceUsingTestCase): def _load_merge_files(self): - merge_root = self.resourceLocation('merge_sources') + merge_root = helpers.resourceLocation('merge_sources') tests = [] source_ids = collections.defaultdict(list) expected_files = {} diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c12a487a..5d9c7d92 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,13 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import net +from cloudinit import distros from cloudinit.net import cmdline -from cloudinit.net import eni -from cloudinit.net import natural_sort_key -from cloudinit.net import netplan -from cloudinit.net import network_state -from cloudinit.net import renderers -from cloudinit.net import sysconfig +from cloudinit.net import ( + eni, interface_has_own_mac, natural_sort_key, netplan, network_state, + renderers, sysconfig) from cloudinit.sources.helpers import openstack from cloudinit import temp_utils from cloudinit import util @@ -132,7 +130,40 @@ OS_SAMPLES = [ 'in_macs': { 'fa:16:3e:ed:9a:59': 'eth0', }, - 'out_sysconfig': [ + 'out_sysconfig_opensuse': [ + ('etc/sysconfig/network/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))], + 'out_sysconfig_rhel': [ ('etc/sysconfig/network-scripts/ifcfg-eth0', """ # Created by cloud-init on instance boot automatically, do not edit. @@ -165,6 +196,7 @@ dns = none ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + }, { 'in_data': { @@ -198,7 +230,42 @@ dns = none 'in_macs': { 'fa:16:3e:ed:9a:59': 'eth0', }, - 'out_sysconfig': [ + 'out_sysconfig_opensuse': [ + ('etc/sysconfig/network/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +IPADDR1=10.0.0.10 +NETMASK=255.255.252.0 +NETMASK1=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))], + 'out_sysconfig_rhel': [ ('etc/sysconfig/network-scripts/ifcfg-eth0', """ # Created by cloud-init on instance boot automatically, do not edit. @@ -233,6 +300,7 @@ dns = none ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + }, { 'in_data': { @@ -286,7 +354,44 @@ dns = none 'in_macs': { 'fa:16:3e:ed:9a:59': 'eth0', }, - 'out_sysconfig': [ + 'out_sysconfig_opensuse': [ + ('etc/sysconfig/network/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +IPV6ADDR=2001:DB8::10/64 +IPV6ADDR_SECONDARIES="2001:DB9::10/64 2001:DB10::10/64" +IPV6INIT=yes +IPV6_DEFAULTGW=2001:DB8::1 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))], + 'out_sysconfig_rhel': [ ('etc/sysconfig/network-scripts/ifcfg-eth0', """ # Created by cloud-init on instance boot automatically, do not edit. @@ -528,6 +633,7 @@ NETWORK_CONFIGS = { config: - type: 'physical' name: 'iface0' + mtu: 8999 subnets: - type: static address: 192.168.14.2/24 @@ -553,6 +659,43 @@ NETWORK_CONFIGS = { """), }, }, + 'dhcpv6_only': { + 'expected_eni': textwrap.dedent("""\ + auto lo + iface lo inet loopback + + auto iface0 + iface iface0 inet6 dhcp + """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + iface0: + dhcp6: true + """).rstrip(' '), + 'yaml': textwrap.dedent("""\ + version: 1 + config: + - type: 'physical' + name: 'iface0' + subnets: + - {'type': 'dhcp6'} + """).rstrip(' '), + 'expected_sysconfig': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=iface0 + DHCPV6C=yes + IPV6INIT=yes + DEVICE=iface0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + }, + }, 'all': { 'expected_eni': ("""\ auto lo @@ -608,6 +751,7 @@ iface br0 inet static bridge_stp off bridge_waitport 1 eth3 bridge_waitport 2 eth4 + hwaddress bb:bb:bb:bb:bb:aa # control-alias br0 iface br0 inet6 static @@ -626,8 +770,8 @@ iface eth0.101 inet static dns-nameservers 192.168.0.10 10.23.23.134 dns-search barley.maas sacchromyces.maas brettanomyces.maas gateway 192.168.0.1 - hwaddress aa:bb:cc:dd:ee:11 mtu 1500 + hwaddress aa:bb:cc:dd:ee:11 vlan-raw-device eth0 vlan_id 101 @@ -673,6 +817,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true interfaces: - eth1 - eth2 + macaddress: aa:bb:cc:dd:ee:ff parameters: mii-monitor-interval: 100 mode: active-backup @@ -685,6 +830,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true interfaces: - eth3 - eth4 + macaddress: bb:bb:bb:bb:bb:aa nameservers: addresses: - 8.8.8.8 @@ -723,6 +869,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true id: 101 link: eth0 macaddress: aa:bb:cc:dd:ee:11 + mtu: 1500 nameservers: addresses: - 192.168.0.10 @@ -740,7 +887,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true """miimon=100" BONDING_SLAVE0=eth1 BONDING_SLAVE1=eth2 - BOOTPROTO=dhcp + BOOTPROTO=none DEVICE=bond0 DHCPV6C=yes IPV6INIT=yes @@ -767,6 +914,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true IPV6ADDR=2001:1::1/64 IPV6INIT=yes IPV6_DEFAULTGW=2001:4800:78ff:1b::1 + MACADDR=bb:bb:bb:bb:bb:aa NETMASK=255.255.255.0 NM_CONTROLLED=no ONBOOT=yes @@ -886,6 +1034,8 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true mtu: 1500 subnets: - type: static + # When 'mtu' matches device-level mtu, no warnings + mtu: 1500 address: 192.168.0.2/24 gateway: 192.168.0.1 dns_nameservers: @@ -935,6 +1085,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true use_tempaddr: 1 forwarding: 1 # basically anything in /proc/sys/net/ipv6/conf/.../ + mac_address: bb:bb:bb:bb:bb:aa params: bridge_ageing: 250 bridge_bridgeprio: 22 @@ -994,6 +1145,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - type: bond name: bond0 mac_address: "aa:bb:cc:dd:e8:ff" + mtu: 9000 bond_interfaces: - bond0s0 - bond0s1 @@ -1036,6 +1188,8 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true interfaces: - bond0s0 - bond0s1 + macaddress: aa:bb:cc:dd:e8:ff + mtu: 9000 parameters: mii-monitor-interval: 100 mode: active-backup @@ -1108,7 +1262,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true version: 2 """), - 'expected_sysconfig': { + 'expected_sysconfig_opensuse': { 'ifcfg-bond0': textwrap.dedent("""\ BONDING_MASTER=yes BONDING_OPTS="mode=active-backup xmit_hash_policy=layer3+4 miimon=100" @@ -1123,6 +1277,59 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true IPADDR1=192.168.1.2 IPV6ADDR=2001:1::1/92 IPV6INIT=yes + MTU=9000 + NETMASK=255.255.255.0 + NETMASK1=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Bond + USERCTL=no + """), + 'ifcfg-bond0s0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=bond0s0 + HWADDR=aa:bb:cc:dd:e8:00 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no + """), + 'ifroute-bond0': textwrap.dedent("""\ + ADDRESS0=10.1.3.0 + GATEWAY0=192.168.0.3 + NETMASK0=255.255.255.0 + """), + 'ifcfg-bond0s1': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=bond0s1 + HWADDR=aa:bb:cc:dd:e8:01 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no + """), + }, + + 'expected_sysconfig_rhel': { + 'ifcfg-bond0': textwrap.dedent("""\ + BONDING_MASTER=yes + BONDING_OPTS="mode=active-backup xmit_hash_policy=layer3+4 miimon=100" + BONDING_SLAVE0=bond0s0 + BONDING_SLAVE1=bond0s1 + BOOTPROTO=none + DEFROUTE=yes + DEVICE=bond0 + GATEWAY=192.168.0.1 + MACADDR=aa:bb:cc:dd:e8:ff + IPADDR=192.168.0.2 + IPADDR1=192.168.1.2 + IPV6ADDR=2001:1::1/92 + IPV6INIT=yes + MTU=9000 NETMASK=255.255.255.0 NETMASK1=255.255.255.0 NM_CONTROLLED=no @@ -1169,6 +1376,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true name: en0 mac_address: "aa:bb:cc:dd:e8:00" - type: vlan + mtu: 2222 name: en0.99 vlan_link: en0 vlan_id: 99 @@ -1204,6 +1412,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true IPV6ADDR=2001:1::bbbb/96 IPV6INIT=yes IPV6_DEFAULTGW=2001:1::1 + MTU=2222 NETMASK=255.255.255.0 NETMASK1=255.255.255.0 NM_CONTROLLED=no @@ -1405,6 +1614,7 @@ DEFAULT_DEV_ATTRS = { "address": "07-1C-C6-75-A4-BE", "device/driver": None, "device/device": None, + "name_assign_type": "4", } } @@ -1443,6 +1653,12 @@ def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, class TestGenerateFallbackConfig(CiTestCase): + def setUp(self): + super(TestGenerateFallbackConfig, self).setUp() + self.add_patch( + "cloudinit.util.get_cmdline", "m_get_cmdline", + return_value="root=/dev/sda1") + @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") @@ -1452,11 +1668,14 @@ class TestGenerateFallbackConfig(CiTestCase): 'eth0': { 'bridge': False, 'carrier': False, 'dormant': False, 'operstate': 'down', 'address': '00:11:22:33:44:55', - 'device/driver': 'hv_netsvc', 'device/device': '0x3'}, + 'device/driver': 'hv_netsvc', 'device/device': '0x3', + 'name_assign_type': '4'}, 'eth1': { 'bridge': False, 'carrier': False, 'dormant': False, 'operstate': 'down', 'address': '00:11:22:33:44:55', - 'device/driver': 'mlx4_core', 'device/device': '0x7'}, + 'device/driver': 'mlx4_core', 'device/device': '0x7', + 'name_assign_type': '4'}, + } tmp_dir = self.tmp_dir() @@ -1474,7 +1693,7 @@ class TestGenerateFallbackConfig(CiTestCase): # don't set rulepath so eni writes them renderer = eni.Renderer( {'eni_path': 'interfaces', 'netrules_path': 'netrules'}) - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) @@ -1512,11 +1731,13 @@ iface eth0 inet dhcp 'eth1': { 'bridge': False, 'carrier': False, 'dormant': False, 'operstate': 'down', 'address': '00:11:22:33:44:55', - 'device/driver': 'hv_netsvc', 'device/device': '0x3'}, + 'device/driver': 'hv_netsvc', 'device/device': '0x3', + 'name_assign_type': '4'}, 'eth0': { 'bridge': False, 'carrier': False, 'dormant': False, 'operstate': 'down', 'address': '00:11:22:33:44:55', - 'device/driver': 'mlx4_core', 'device/device': '0x7'}, + 'device/driver': 'mlx4_core', 'device/device': '0x7', + 'name_assign_type': '4'}, } tmp_dir = self.tmp_dir() @@ -1536,7 +1757,7 @@ iface eth0 inet dhcp # don't set rulepath so eni writes them renderer = eni.Renderer( {'eni_path': 'interfaces', 'netrules_path': 'netrules'}) - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) @@ -1565,13 +1786,83 @@ iface eth1 inet dhcp ] self.assertEqual(", ".join(expected_rule) + '\n', contents.lstrip()) + @mock.patch("cloudinit.util.get_cmdline") + @mock.patch("cloudinit.util.udevadm_settle") + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_unstable_names(self, mock_get_devicelist, mock_read_sys_net, + mock_sys_dev_path, mock_settle, m_get_cmdline): + """verify that udevadm settle is called when we find unstable names""" + devices = { + 'eth0': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'hv_netsvc', 'device/device': '0x3', + 'name_assign_type': False}, + 'ens4': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'mlx4_core', 'device/device': '0x7', + 'name_assign_type': '4'}, -class TestSysConfigRendering(CiTestCase): + } + + m_get_cmdline.return_value = '' + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path, + dev_attrs=devices) + net.generate_fallback_config(config_driver=True) + self.assertEqual(1, mock_settle.call_count) + + @mock.patch("cloudinit.util.get_cmdline") + @mock.patch("cloudinit.util.udevadm_settle") + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_unstable_names_disabled(self, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path, + mock_settle, m_get_cmdline): + """verify udevadm settle not called when cmdline has net.ifnames=0""" + devices = { + 'eth0': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'hv_netsvc', 'device/device': '0x3', + 'name_assign_type': False}, + 'ens4': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'mlx4_core', 'device/device': '0x7', + 'name_assign_type': '4'}, + + } + + m_get_cmdline.return_value = 'net.ifnames=0' + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path, + dev_attrs=devices) + net.generate_fallback_config(config_driver=True) + self.assertEqual(0, mock_settle.call_count) + + +class TestRhelSysConfigRendering(CiTestCase): + + with_logs = True scripts_dir = '/etc/sysconfig/network-scripts' header = ('# Created by cloud-init on instance boot automatically, ' 'do not edit.\n#\n') + expected_name = 'expected_sysconfig' + + def _get_renderer(self): + distro_cls = distros.fetch('rhel') + return sysconfig.Renderer( + config=distro_cls.renderer_configs.get('sysconfig')) + def _render_and_read(self, network_config=None, state=None, dir=None): if dir is None: dir = self.tmp_dir() @@ -1583,8 +1874,8 @@ class TestSysConfigRendering(CiTestCase): else: raise ValueError("Expected data or state, got neither") - renderer = sysconfig.Renderer() - renderer.render_network_state(ns, dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=dir) return dir2dict(dir) def _compare_files_to_expected(self, expected, found): @@ -1610,12 +1901,13 @@ class TestSysConfigRendering(CiTestCase): if missing: raise AssertionError("Missing headers in: %s" % missing) + @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot") @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") def test_default_generation(self, mock_get_devicelist, mock_read_sys_net, - mock_sys_dev_path): + mock_sys_dev_path, m_get_cmdline): tmp_dir = self.tmp_dir() _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path) @@ -1627,8 +1919,8 @@ class TestSysConfigRendering(CiTestCase): render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) - renderer = sysconfig.Renderer() - renderer.render_network_state(ns, render_dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000' with open(os.path.join(render_dir, render_file)) as fh: @@ -1679,9 +1971,9 @@ USERCTL=no network_cfg = openstack.convert_net_json(net_json, known_macs=macs) ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) - renderer = sysconfig.Renderer() + renderer = self._get_renderer() with self.assertRaises(ValueError): - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertEqual([], os.listdir(render_dir)) def test_multiple_ipv6_default_gateways(self): @@ -1717,9 +2009,9 @@ USERCTL=no network_cfg = openstack.convert_net_json(net_json, known_macs=macs) ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) - renderer = sysconfig.Renderer() + renderer = self._get_renderer() with self.assertRaises(ValueError): - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertEqual([], os.listdir(render_dir)) def test_openstack_rendering_samples(self): @@ -1731,12 +2023,13 @@ USERCTL=no ex_input, known_macs=ex_mac_addrs) ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) - renderer = sysconfig.Renderer() + renderer = self._get_renderer() # render a multiple times to simulate reboots - renderer.render_network_state(ns, render_dir) - renderer.render_network_state(ns, render_dir) - renderer.render_network_state(ns, render_dir) - for fn, expected_content in os_sample.get('out_sysconfig', []): + renderer.render_network_state(ns, target=render_dir) + renderer.render_network_state(ns, target=render_dir) + renderer.render_network_state(ns, target=render_dir) + for fn, expected_content in os_sample.get('out_sysconfig_rhel', + []): with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) @@ -1744,8 +2037,8 @@ USERCTL=no ns = network_state.parse_net_config_data(CONFIG_V1_SIMPLE_SUBNET) render_dir = self.tmp_path("render") os.makedirs(render_dir) - renderer = sysconfig.Renderer() - renderer.render_network_state(ns, render_dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) found = dir2dict(render_dir) nspath = '/etc/sysconfig/network-scripts/' self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) @@ -1770,8 +2063,8 @@ USERCTL=no ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) render_dir = self.tmp_path("render") os.makedirs(render_dir) - renderer = sysconfig.Renderer() - renderer.render_network_state(ns, render_dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) found = dir2dict(render_dir) nspath = '/etc/sysconfig/network-scripts/' self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) @@ -1788,56 +2081,369 @@ USERCTL=no self.assertEqual(expected, found[nspath + 'ifcfg-eth0']) def test_bond_config(self): + expected_name = 'expected_sysconfig_rhel' + entry = NETWORK_CONFIGS['bond'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[expected_name], found) + self._assert_headers(found) + + def test_vlan_config(self): + entry = NETWORK_CONFIGS['vlan'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_bridge_config(self): + entry = NETWORK_CONFIGS['bridge'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_manual_config(self): + entry = NETWORK_CONFIGS['manual'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_all_config(self): + entry = NETWORK_CONFIGS['all'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + self.assertNotIn( + 'WARNING: Network config: ignoring eth0.101 device-level mtu', + self.logs.getvalue()) + + def test_small_config(self): + entry = NETWORK_CONFIGS['small'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + def test_v4_and_v6_static_config(self): + entry = NETWORK_CONFIGS['v4_and_v6_static'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + expected_msg = ( + 'WARNING: Network config: ignoring iface0 device-level mtu:8999' + ' because ipv4 subnet-level mtu:9000 provided.') + self.assertIn(expected_msg, self.logs.getvalue()) + + def test_dhcpv6_only_config(self): + entry = NETWORK_CONFIGS['dhcpv6_only'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + + +class TestOpenSuseSysConfigRendering(CiTestCase): + + with_logs = True + + scripts_dir = '/etc/sysconfig/network' + header = ('# Created by cloud-init on instance boot automatically, ' + 'do not edit.\n#\n') + + expected_name = 'expected_sysconfig' + + def _get_renderer(self): + distro_cls = distros.fetch('opensuse') + return sysconfig.Renderer( + config=distro_cls.renderer_configs.get('sysconfig')) + + def _render_and_read(self, network_config=None, state=None, dir=None): + if dir is None: + dir = self.tmp_dir() + + if network_config: + ns = network_state.parse_net_config_data(network_config) + elif state: + ns = state + else: + raise ValueError("Expected data or state, got neither") + + renderer = self._get_renderer() + renderer.render_network_state(ns, target=dir) + return dir2dict(dir) + + def _compare_files_to_expected(self, expected, found): + orig_maxdiff = self.maxDiff + expected_d = dict( + (os.path.join(self.scripts_dir, k), util.load_shell_content(v)) + for k, v in expected.items()) + + # only compare the files in scripts_dir + scripts_found = dict( + (k, util.load_shell_content(v)) for k, v in found.items() + if k.startswith(self.scripts_dir)) + try: + self.maxDiff = None + self.assertEqual(expected_d, scripts_found) + finally: + self.maxDiff = orig_maxdiff + + def _assert_headers(self, found): + missing = [f for f in found + if (f.startswith(self.scripts_dir) and + not found[f].startswith(self.header))] + if missing: + raise AssertionError("Missing headers in: %s" % missing) + + @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot") + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_read_sys_net, + mock_sys_dev_path, m_get_cmdline): + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) + + render_file = 'etc/sysconfig/network/ifcfg-eth1000' + with open(os.path.join(render_dir, render_file)) as fh: + content = fh.read() + expected_content = """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1000 +HWADDR=07-1C-C6-75-A4-BE +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip() + self.assertEqual(expected_content, content) + + def test_multiple_ipv4_default_gateways(self): + """ValueError is raised when duplicate ipv4 gateways exist.""" + net_json = { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }, { + "netmask": "0.0.0.0", # A second default gateway + "network": "0.0.0.0", + "gateway": "172.20.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + } + macs = {'fa:16:3e:ed:9a:59': 'eth0'} + render_dir = self.tmp_dir() + network_cfg = openstack.convert_net_json(net_json, known_macs=macs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = self._get_renderer() + with self.assertRaises(ValueError): + renderer.render_network_state(ns, target=render_dir) + self.assertEqual([], os.listdir(render_dir)) + + def test_multiple_ipv6_default_gateways(self): + """ValueError is raised when duplicate ipv6 gateways exist.""" + net_json = { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "public-ipv6", + "type": "ipv6", "netmask": "", + "link": "tap1a81968a-79", + "routes": [{ + "gateway": "2001:DB8::1", + "netmask": "::", + "network": "::" + }, { + "gateway": "2001:DB9::1", + "netmask": "::", + "network": "::" + }], + "ip_address": "2001:DB8::10", "id": "network1" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + } + macs = {'fa:16:3e:ed:9a:59': 'eth0'} + render_dir = self.tmp_dir() + network_cfg = openstack.convert_net_json(net_json, known_macs=macs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = self._get_renderer() + with self.assertRaises(ValueError): + renderer.render_network_state(ns, target=render_dir) + self.assertEqual([], os.listdir(render_dir)) + + def test_openstack_rendering_samples(self): + for os_sample in OS_SAMPLES: + render_dir = self.tmp_dir() + ex_input = os_sample['in_data'] + ex_mac_addrs = os_sample['in_macs'] + network_cfg = openstack.convert_net_json( + ex_input, known_macs=ex_mac_addrs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = self._get_renderer() + # render a multiple times to simulate reboots + renderer.render_network_state(ns, target=render_dir) + renderer.render_network_state(ns, target=render_dir) + renderer.render_network_state(ns, target=render_dir) + for fn, expected_content in os_sample.get('out_sysconfig_opensuse', + []): + with open(os.path.join(render_dir, fn)) as fh: + self.assertEqual(expected_content, fh.read()) + + def test_network_config_v1_samples(self): + ns = network_state.parse_net_config_data(CONFIG_V1_SIMPLE_SUBNET) + render_dir = self.tmp_path("render") + os.makedirs(render_dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) + found = dir2dict(render_dir) + nspath = '/etc/sysconfig/network/' + self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) + expected = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEFROUTE=yes +DEVICE=interface0 +GATEWAY=10.0.2.2 +HWADDR=52:54:00:12:34:00 +IPADDR=10.0.2.15 +NETMASK=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + self.assertEqual(expected, found[nspath + 'ifcfg-interface0']) + + def test_config_with_explicit_loopback(self): + ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) + render_dir = self.tmp_path("render") + os.makedirs(render_dir) + renderer = self._get_renderer() + renderer.render_network_state(ns, target=render_dir) + found = dir2dict(render_dir) + nspath = '/etc/sysconfig/network/' + self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) + expected = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + self.assertEqual(expected, found[nspath + 'ifcfg-eth0']) + + def test_bond_config(self): + expected_name = 'expected_sysconfig_opensuse' entry = NETWORK_CONFIGS['bond'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + for fname, contents in entry[expected_name].items(): + print(fname) + print(contents) + print() + print('-- expected ^ | v rendered --') + for fname, contents in found.items(): + print(fname) + print(contents) + print() + self._compare_files_to_expected(entry[expected_name], found) self._assert_headers(found) def test_vlan_config(self): entry = NETWORK_CONFIGS['vlan'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) def test_bridge_config(self): entry = NETWORK_CONFIGS['bridge'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) def test_manual_config(self): entry = NETWORK_CONFIGS['manual'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) def test_all_config(self): entry = NETWORK_CONFIGS['all'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) + self.assertNotIn( + 'WARNING: Network config: ignoring eth0.101 device-level mtu', + self.logs.getvalue()) def test_small_config(self): entry = NETWORK_CONFIGS['small'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) def test_v4_and_v6_static_config(self): entry = NETWORK_CONFIGS['v4_and_v6_static'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) - self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + expected_msg = ( + 'WARNING: Network config: ignoring iface0 device-level mtu:8999' + ' because ipv4 subnet-level mtu:9000 provided.') + self.assertIn(expected_msg, self.logs.getvalue()) + + def test_dhcpv6_only_config(self): + entry = NETWORK_CONFIGS['dhcpv6_only'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) class TestEniNetRendering(CiTestCase): + @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot") @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") def test_default_generation(self, mock_get_devicelist, mock_read_sys_net, - mock_sys_dev_path): + mock_sys_dev_path, m_get_cmdline): tmp_dir = self.tmp_dir() _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path) @@ -1851,7 +2457,7 @@ class TestEniNetRendering(CiTestCase): renderer = eni.Renderer( {'eni_path': 'interfaces', 'netrules_path': None}) - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) @@ -1871,7 +2477,7 @@ iface eth1000 inet dhcp tmp_dir = self.tmp_dir() ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) renderer = eni.Renderer() - renderer.render_network_state(ns, tmp_dir) + renderer.render_network_state(ns, target=tmp_dir) expected = """\ auto lo iface lo inet loopback @@ -1885,6 +2491,7 @@ iface eth0 inet dhcp class TestNetplanNetRendering(CiTestCase): + @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot") @mock.patch("cloudinit.net.netplan._clean_default") @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @@ -1892,7 +2499,7 @@ class TestNetplanNetRendering(CiTestCase): def test_default_generation(self, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path, - mock_clean_default): + mock_clean_default, m_get_cmdline): tmp_dir = self.tmp_dir() _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path) @@ -1907,7 +2514,7 @@ class TestNetplanNetRendering(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': False}) - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, render_target))) @@ -2012,7 +2619,7 @@ class TestNetplanPostcommands(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) mock_netplan_generate.assert_called_with(run=True) mock_net_setup_link.assert_called_with(run=True) @@ -2037,7 +2644,7 @@ class TestNetplanPostcommands(CiTestCase): '/sys/class/net/lo'], capture=True), ] with mock.patch.object(os.path, 'islink', return_value=True): - renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, target=render_dir) mock_subp.assert_has_calls(expected) @@ -2232,7 +2839,7 @@ class TestNetplanRoundTrip(CiTestCase): renderer = netplan.Renderer( config={'netplan_path': netplan_path}) - renderer.render_network_state(ns, target) + renderer.render_network_state(ns, target=target) return dir2dict(target) def testsimple_render_bond_netplan(self): @@ -2277,6 +2884,13 @@ class TestNetplanRoundTrip(CiTestCase): entry['expected_netplan'].splitlines(), files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_dhcpv6_only(self): + entry = NETWORK_CONFIGS['dhcpv6_only'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_all(self): entry = NETWORK_CONFIGS['all'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) @@ -2296,6 +2910,7 @@ class TestNetplanRoundTrip(CiTestCase): class TestEniRoundTrip(CiTestCase): + def _render_and_read(self, network_config=None, state=None, eni_path=None, netrules_path=None, dir=None): if dir is None: @@ -2314,7 +2929,7 @@ class TestEniRoundTrip(CiTestCase): renderer = eni.Renderer( config={'eni_path': eni_path, 'netrules_path': netrules_path}) - renderer.render_network_state(ns, dir) + renderer.render_network_state(ns, target=dir) return dir2dict(dir) def testsimple_convert_and_render(self): @@ -2345,6 +2960,13 @@ class TestEniRoundTrip(CiTestCase): entry['expected_eni'].splitlines(), files['/etc/network/interfaces'].splitlines()) + def testsimple_render_dhcpv6_only(self): + entry = NETWORK_CONFIGS['dhcpv6_only'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_eni'].splitlines(), + files['/etc/network/interfaces'].splitlines()) + def testsimple_render_v4_and_v6_static(self): entry = NETWORK_CONFIGS['v4_and_v6_static'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) @@ -2569,6 +3191,43 @@ class TestGetInterfaces(CiTestCase): any_order=True) +class TestInterfaceHasOwnMac(CiTestCase): + """Test interface_has_own_mac. This is admittedly a bit whitebox.""" + + @mock.patch('cloudinit.net.read_sys_net_int', return_value=None) + def test_non_strict_with_no_addr_assign_type(self, m_read_sys_net_int): + """If nic does not have addr_assign_type, it is not "stolen". + + SmartOS containers do not provide the addr_assign_type in /sys. + + $ ( cd /sys/class/net/eth0/ && grep -r . *) + address:90:b8:d0:20:e1:b0 + addr_len:6 + flags:0x1043 + ifindex:2 + mtu:1500 + tx_queue_len:1 + type:1 + """ + self.assertTrue(interface_has_own_mac("eth0")) + + @mock.patch('cloudinit.net.read_sys_net_int', return_value=None) + def test_strict_with_no_addr_assign_type_raises(self, m_read_sys_net_int): + with self.assertRaises(ValueError): + interface_has_own_mac("eth0", True) + + @mock.patch('cloudinit.net.read_sys_net_int') + def test_expected_values(self, m_read_sys_net_int): + msg = "address_assign_type=%d said to not have own mac" + for address_assign_type in (0, 1, 3): + m_read_sys_net_int.return_value = address_assign_type + self.assertTrue( + interface_has_own_mac("eth0", msg % address_assign_type)) + + m_read_sys_net_int.return_value = 2 + self.assertFalse(interface_has_own_mac("eth0")) + + class TestGetInterfacesByMac(CiTestCase): _data = {'bonds': ['bond1'], 'bridges': ['bridge1'], @@ -2601,11 +3260,15 @@ class TestGetInterfacesByMac(CiTestCase): def _se_interface_has_own_mac(self, name): return name in self.data['own_macs'] + def _se_get_ib_interface_hwaddr(self, name, ethernet_format): + ib_hwaddr = self.data.get('ib_hwaddr', {}) + return ib_hwaddr.get(name, {}).get(ethernet_format) + def _mock_setup(self): self.data = copy.deepcopy(self._data) self.data['devices'] = set(list(self.data['macs'].keys())) mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge', - 'interface_has_own_mac', 'is_vlan') + 'interface_has_own_mac', 'is_vlan', 'get_ib_interface_hwaddr') self.mocks = {} for n in mocks: m = mock.patch('cloudinit.net.' + n, @@ -2679,6 +3342,20 @@ class TestGetInterfacesByMac(CiTestCase): ret = net.get_interfaces_by_mac() self.assertEqual('lo', ret[empty_mac]) + def test_ib(self): + ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56' + ib_addr_eth_format = '00:11:22:33:44:56' + self._mock_setup() + self.data['devices'] = ['enp0s1', 'ib0'] + self.data['own_macs'].append('ib0') + self.data['macs']['ib0'] = ib_addr + self.data['ib_hwaddr'] = {'ib0': {True: ib_addr_eth_format, + False: ib_addr}} + result = net.get_interfaces_by_mac() + expected = {'aa:aa:aa:aa:aa:01': 'enp0s1', + ib_addr_eth_format: 'ib0', ib_addr: 'ib0'} + self.assertEqual(expected, result) + class TestInterfacesSorting(CiTestCase): @@ -2693,6 +3370,67 @@ class TestInterfacesSorting(CiTestCase): ['enp0s3', 'enp0s8', 'enp0s13', 'enp1s2', 'enp2s0', 'enp2s3']) +class TestGetIBHwaddrsByInterface(CiTestCase): + + _ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56' + _ib_addr_eth_format = '00:11:22:33:44:56' + _data = {'devices': ['enp0s1', 'enp0s2', 'bond1', 'bridge1', + 'bridge1-nic', 'tun0', 'ib0'], + 'bonds': ['bond1'], + 'bridges': ['bridge1'], + 'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1', 'ib0'], + 'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01', + 'enp0s2': 'aa:aa:aa:aa:aa:02', + 'bond1': 'aa:aa:aa:aa:aa:01', + 'bridge1': 'aa:aa:aa:aa:aa:03', + 'bridge1-nic': 'aa:aa:aa:aa:aa:03', + 'tun0': None, + 'ib0': _ib_addr}, + 'ib_hwaddr': {'ib0': {True: _ib_addr_eth_format, + False: _ib_addr}}} + data = {} + + def _mock_setup(self): + self.data = copy.deepcopy(self._data) + mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge', + 'interface_has_own_mac', 'get_ib_interface_hwaddr') + self.mocks = {} + for n in mocks: + m = mock.patch('cloudinit.net.' + n, + side_effect=getattr(self, '_se_' + n)) + self.addCleanup(m.stop) + self.mocks[n] = m.start() + + def _se_get_devicelist(self): + return self.data['devices'] + + def _se_get_interface_mac(self, name): + return self.data['macs'][name] + + def _se_is_bridge(self, name): + return name in self.data['bridges'] + + def _se_interface_has_own_mac(self, name): + return name in self.data['own_macs'] + + def _se_get_ib_interface_hwaddr(self, name, ethernet_format): + ib_hwaddr = self.data.get('ib_hwaddr', {}) + return ib_hwaddr.get(name, {}).get(ethernet_format) + + def test_ethernet(self): + self._mock_setup() + self.data['devices'].remove('ib0') + result = net.get_ib_hwaddrs_by_interface() + expected = {} + self.assertEqual(expected, result) + + def test_ib(self): + self._mock_setup() + result = net.get_ib_hwaddrs_by_interface() + expected = {'ib0': self._ib_addr} + self.assertEqual(expected, result) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py new file mode 100644 index 00000000..2e64c6c7 --- /dev/null +++ b/tests/unittests/test_reporting_hyperv.py @@ -0,0 +1,134 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.reporting import events +from cloudinit.reporting import handlers + +import json +import os + +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase + + +class TestKvpEncoding(CiTestCase): + def test_encode_decode(self): + kvp = {'key': 'key1', 'value': 'value1'} + kvp_reporting = handlers.HyperVKvpReportingHandler() + data = kvp_reporting._encode_kvp_item(kvp['key'], kvp['value']) + self.assertEqual(len(data), kvp_reporting.HV_KVP_RECORD_SIZE) + decoded_kvp = kvp_reporting._decode_kvp_item(data) + self.assertEqual(kvp, decoded_kvp) + + +class TextKvpReporter(CiTestCase): + def setUp(self): + super(TextKvpReporter, self).setUp() + self.tmp_file_path = self.tmp_path('kvp_pool_file') + util.ensure_file(self.tmp_file_path) + + def test_event_type_can_be_filtered(self): + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path, + event_types=['foo', 'bar']) + + reporter.publish_event( + events.ReportingEvent('foo', 'name', 'description')) + reporter.publish_event( + events.ReportingEvent('some_other', 'name', 'description3')) + reporter.q.join() + + kvps = list(reporter._iterate_kvps(0)) + self.assertEqual(1, len(kvps)) + + reporter.publish_event( + events.ReportingEvent('bar', 'name', 'description2')) + reporter.q.join() + kvps = list(reporter._iterate_kvps(0)) + self.assertEqual(2, len(kvps)) + + self.assertIn('foo', kvps[0]['key']) + self.assertIn('bar', kvps[1]['key']) + self.assertNotIn('some_other', kvps[0]['key']) + self.assertNotIn('some_other', kvps[1]['key']) + + def test_events_are_over_written(self): + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + + self.assertEqual(0, len(list(reporter._iterate_kvps(0)))) + + reporter.publish_event( + events.ReportingEvent('foo', 'name1', 'description')) + reporter.publish_event( + events.ReportingEvent('foo', 'name2', 'description')) + reporter.q.join() + self.assertEqual(2, len(list(reporter._iterate_kvps(0)))) + + reporter2 = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + reporter2.incarnation_no = reporter.incarnation_no + 1 + reporter2.publish_event( + events.ReportingEvent('foo', 'name3', 'description')) + reporter2.q.join() + + self.assertEqual(2, len(list(reporter2._iterate_kvps(0)))) + + def test_events_with_higher_incarnation_not_over_written(self): + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + + self.assertEqual(0, len(list(reporter._iterate_kvps(0)))) + + reporter.publish_event( + events.ReportingEvent('foo', 'name1', 'description')) + reporter.publish_event( + events.ReportingEvent('foo', 'name2', 'description')) + reporter.q.join() + self.assertEqual(2, len(list(reporter._iterate_kvps(0)))) + + reporter3 = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + reporter3.incarnation_no = reporter.incarnation_no - 1 + reporter3.publish_event( + events.ReportingEvent('foo', 'name3', 'description')) + reporter3.q.join() + self.assertEqual(3, len(list(reporter3._iterate_kvps(0)))) + + def test_finish_event_result_is_logged(self): + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + reporter.publish_event( + events.FinishReportingEvent('name2', 'description1', + result=events.status.FAIL)) + reporter.q.join() + self.assertIn('FAIL', list(reporter._iterate_kvps(0))[0]['value']) + + def test_file_operation_issue(self): + os.remove(self.tmp_file_path) + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + reporter.publish_event( + events.FinishReportingEvent('name2', 'description1', + result=events.status.FAIL)) + reporter.q.join() + + def test_event_very_long(self): + reporter = handlers.HyperVKvpReportingHandler( + kvp_file_path=self.tmp_file_path) + description = 'ab' * reporter.HV_KVP_EXCHANGE_MAX_VALUE_SIZE + long_event = events.FinishReportingEvent( + 'event_name', + description, + result=events.status.FAIL) + reporter.publish_event(long_event) + reporter.q.join() + kvps = list(reporter._iterate_kvps(0)) + self.assertEqual(3, len(kvps)) + + # restore from the kvp to see the content are all there + full_description = '' + for i in range(len(kvps)): + msg_slice = json.loads(kvps[i]['value']) + self.assertEqual(msg_slice['msg_i'], i) + full_description += msg_slice['msg'] + self.assertEqual(description, full_description) diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 22718108..4cd27eed 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -8,10 +8,16 @@ import logging from cloudinit.config import cc_rh_subscription from cloudinit import util -from cloudinit.tests.helpers import TestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock +SUBMGR = cc_rh_subscription.SubscriptionManager +SUB_MAN_CLI = 'cloudinit.config.cc_rh_subscription._sub_man_cli' + + +@mock.patch(SUB_MAN_CLI) +class GoodTests(CiTestCase): + with_logs = True -class GoodTests(TestCase): def setUp(self): super(GoodTests, self).setUp() self.name = "cc_rh_subscription" @@ -19,7 +25,6 @@ class GoodTests(TestCase): self.log = logging.getLogger("good_tests") self.args = [] self.handle = cc_rh_subscription.handle - self.SM = cc_rh_subscription.SubscriptionManager self.config = {'rh_subscription': {'username': 'scooby@do.com', @@ -35,55 +40,47 @@ class GoodTests(TestCase): 'disable-repo': ['repo4', 'repo5'] }} - def test_already_registered(self): + def test_already_registered(self, m_sman_cli): ''' Emulates a system that is already registered. Ensure it gets a non-ProcessExecution error from is_registered() ''' - with mock.patch.object(cc_rh_subscription.SubscriptionManager, - '_sub_man_cli') as mockobj: - self.SM.log_success = mock.MagicMock() - self.handle(self.name, self.config, self.cloud_init, - self.log, self.args) - self.assertEqual(self.SM.log_success.call_count, 1) - self.assertEqual(mockobj.call_count, 1) - - def test_simple_registration(self): + self.handle(self.name, self.config, self.cloud_init, + self.log, self.args) + self.assertEqual(m_sman_cli.call_count, 1) + self.assertIn('System is already registered', self.logs.getvalue()) + + def test_simple_registration(self, m_sman_cli): ''' Simple registration with username and password ''' - self.SM.log_success = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (reg, 'bar')]) + m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')] self.handle(self.name, self.config, self.cloud_init, self.log, self.args) - self.assertIn(mock.call(['identity']), - self.SM._sub_man_cli.call_args_list) + self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list) self.assertIn(mock.call(['register', '--username=scooby@do.com', '--password=scooby-snacks'], logstring_val=True), - self.SM._sub_man_cli.call_args_list) - - self.assertEqual(self.SM.log_success.call_count, 1) - self.assertEqual(self.SM._sub_man_cli.call_count, 2) + m_sman_cli.call_args_list) + self.assertIn('rh_subscription plugin completed successfully', + self.logs.getvalue()) + self.assertEqual(m_sman_cli.call_count, 2) @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos") - @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_sub_man_cli") - def test_update_repos_disable_with_none(self, m_sub_man_cli, m_get_repos): + def test_update_repos_disable_with_none(self, m_get_repos, m_sman_cli): cfg = copy.deepcopy(self.config) m_get_repos.return_value = ([], ['repo1']) - m_sub_man_cli.return_value = (b'', b'') cfg['rh_subscription'].update( {'enable-repo': ['repo1'], 'disable-repo': None}) mysm = cc_rh_subscription.SubscriptionManager(cfg) self.assertEqual(True, mysm.update_repos()) m_get_repos.assert_called_with() - self.assertEqual(m_sub_man_cli.call_args_list, + self.assertEqual(m_sman_cli.call_args_list, [mock.call(['repos', '--enable=repo1'])]) - def test_full_registration(self): + def test_full_registration(self, m_sman_cli): ''' Registration with auto-attach, service-level, adding pools, and enabling and disabling yum repos @@ -93,26 +90,28 @@ class GoodTests(TestCase): call_lists.append(['repos', '--disable=repo5', '--enable=repo2', '--enable=repo3']) call_lists.append(['attach', '--auto', '--servicelevel=self-support']) - self.SM.log_success = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (reg, 'bar'), - ('Service level set to: self-support', ''), - ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''), - ('Repo ID: repo1\nRepo ID: repo5\n', ''), - ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: ' - 'repo4', ''), - ('', '')]) + m_sman_cli.side_effect = [ + util.ProcessExecutionError, + (reg, 'bar'), + ('Service level set to: self-support', ''), + ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''), + ('Repo ID: repo1\nRepo ID: repo5\n', ''), + ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: repo4', ''), + ('', '')] self.handle(self.name, self.config_full, self.cloud_init, self.log, self.args) + self.assertEqual(m_sman_cli.call_count, 9) for call in call_lists: - self.assertIn(mock.call(call), self.SM._sub_man_cli.call_args_list) - self.assertEqual(self.SM.log_success.call_count, 1) - self.assertEqual(self.SM._sub_man_cli.call_count, 9) + self.assertIn(mock.call(call), m_sman_cli.call_args_list) + self.assertIn("rh_subscription plugin completed successfully", + self.logs.getvalue()) -class TestBadInput(TestCase): +@mock.patch(SUB_MAN_CLI) +class TestBadInput(CiTestCase): + with_logs = True name = "cc_rh_subscription" cloud_init = None log = logging.getLogger("bad_tests") @@ -155,81 +154,81 @@ class TestBadInput(TestCase): super(TestBadInput, self).setUp() self.handle = cc_rh_subscription.handle - def test_no_password(self): - ''' - Attempt to register without the password key/value - ''' - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + def assert_logged_warnings(self, warnings): + logs = self.logs.getvalue() + missing = [w for w in warnings if "WARNING: " + w not in logs] + self.assertEqual([], missing, "Missing expected warnings.") + + def test_no_password(self, m_sman_cli): + '''Attempt to register without the password key/value.''' + m_sman_cli.side_effect = [util.ProcessExecutionError, + (self.reg, 'bar')] self.handle(self.name, self.config_no_password, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM._sub_man_cli.call_count, 0) + self.assertEqual(m_sman_cli.call_count, 0) - def test_no_org(self): - ''' - Attempt to register without the org key/value - ''' - self.input_is_missing_data(self.config_no_key) - - def test_service_level_without_auto(self): - ''' - Attempt to register using service-level without the auto-attach key - ''' - self.SM.log_warn = mock.MagicMock() - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + def test_no_org(self, m_sman_cli): + '''Attempt to register without the org key/value.''' + m_sman_cli.side_effect = [util.ProcessExecutionError] + self.handle(self.name, self.config_no_key, self.cloud_init, + self.log, self.args) + m_sman_cli.assert_called_with(['identity']) + self.assertEqual(m_sman_cli.call_count, 1) + self.assert_logged_warnings(( + 'Unable to register system due to incomplete information.', + 'Use either activationkey and org *or* userid and password', + 'Registration failed or did not run completely', + 'rh_subscription plugin did not complete successfully')) + + def test_service_level_without_auto(self, m_sman_cli): + '''Attempt to register using service-level without auto-attach key.''' + m_sman_cli.side_effect = [util.ProcessExecutionError, + (self.reg, 'bar')] self.handle(self.name, self.config_service, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM._sub_man_cli.call_count, 1) - self.assertEqual(self.SM.log_warn.call_count, 2) + self.assertEqual(m_sman_cli.call_count, 1) + self.assert_logged_warnings(( + 'The service-level key must be used in conjunction with ', + 'rh_subscription plugin did not complete successfully')) - def test_pool_not_a_list(self): + def test_pool_not_a_list(self, m_sman_cli): ''' Register with pools that are not in the format of a list ''' - self.SM.log_warn = mock.MagicMock() - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + m_sman_cli.side_effect = [util.ProcessExecutionError, + (self.reg, 'bar')] self.handle(self.name, self.config_badpool, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM._sub_man_cli.call_count, 2) - self.assertEqual(self.SM.log_warn.call_count, 2) + self.assertEqual(m_sman_cli.call_count, 2) + self.assert_logged_warnings(( + 'Pools must in the format of a list', + 'rh_subscription plugin did not complete successfully')) - def test_repo_not_a_list(self): + def test_repo_not_a_list(self, m_sman_cli): ''' Register with repos that are not in the format of a list ''' - self.SM.log_warn = mock.MagicMock() - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + m_sman_cli.side_effect = [util.ProcessExecutionError, + (self.reg, 'bar')] self.handle(self.name, self.config_badrepo, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM.log_warn.call_count, 3) - self.assertEqual(self.SM._sub_man_cli.call_count, 2) + self.assertEqual(m_sman_cli.call_count, 2) + self.assert_logged_warnings(( + 'Repo IDs must in the format of a list.', + 'Unable to add or remove repos', + 'rh_subscription plugin did not complete successfully')) - def test_bad_key_value(self): + def test_bad_key_value(self, m_sman_cli): ''' Attempt to register with a key that we don't know ''' - self.SM.log_warn = mock.MagicMock() - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + m_sman_cli.side_effect = [util.ProcessExecutionError, + (self.reg, 'bar')] self.handle(self.name, self.config_badkey, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM.log_warn.call_count, 2) - self.assertEqual(self.SM._sub_man_cli.call_count, 1) - - def input_is_missing_data(self, config): - ''' - Helper def for tests that having missing information - ''' - self.SM.log_warn = mock.MagicMock() - self.SM._sub_man_cli = mock.MagicMock( - side_effect=[util.ProcessExecutionError]) - self.handle(self.name, config, self.cloud_init, - self.log, self.args) - self.SM._sub_man_cli.assert_called_with(['identity']) - self.assertEqual(self.SM.log_warn.call_count, 4) - self.assertEqual(self.SM._sub_man_cli.call_count, 1) + self.assertEqual(m_sman_cli.call_count, 1) + self.assert_logged_warnings(( + 'fookey is not a valid key for rh_subscription. Valid keys are:', + 'rh_subscription plugin did not complete successfully')) # vi: ts=4 expandtab diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 5d3f1ca3..d1ac4942 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -25,7 +25,7 @@ class TestMergeRun(helpers.FilesystemMockingTestCase): 'cloud_init_modules': ['write-files'], 'system_info': {'paths': {'run_dir': new_root}} } - ud = self.readResource('user_data.1.txt') + ud = helpers.readResource('user_data.1.txt') cloud_cfg = util.yaml_dumps(cfg) util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) util.write_file(os.path.join(new_root, 'etc', diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 762974e9..d67c422c 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +import copy import os @@ -127,8 +128,9 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override - self.cfg['unverified_modules'] = ['spacewalk'] # Would have skipped - cloud_cfg = util.yaml_dumps(self.cfg) + cfg = copy.deepcopy(self.cfg) + cfg['unverified_modules'] = ['spacewalk'] # Would have skipped + cloud_cfg = util.yaml_dumps(cfg) util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) util.write_file(os.path.join(self.new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) @@ -150,4 +152,30 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): "running unverified_modules: 'spacewalk'", self.logs.getvalue()) + def test_none_ds_run_with_no_config_modules(self): + """run_section will report no modules run when none are configured.""" + + # re-write cloud.cfg with unverified_modules override + cfg = copy.deepcopy(self.cfg) + # Represent empty configuration in /etc/cloud/cloud.cfg + cfg['cloud_init_modules'] = None + cloud_cfg = util.yaml_dumps(cfg) + util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + util.write_file(os.path.join(self.new_root, 'etc', + 'cloud', 'cloud.cfg'), cloud_cfg) + + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertEqual([], which_ran) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 4c62c8be..73ae897f 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -4,6 +4,7 @@ from mock import patch from cloudinit import ssh_util from cloudinit.tests import helpers as test_helpers +from cloudinit import util VALID_CONTENT = { @@ -56,7 +57,7 @@ TEST_OPTIONS = ( 'user \"root\".\';echo;sleep 10"') -class TestAuthKeyLineParser(test_helpers.TestCase): +class TestAuthKeyLineParser(test_helpers.CiTestCase): def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) @@ -126,7 +127,7 @@ class TestAuthKeyLineParser(test_helpers.TestCase): self.assertFalse(key.valid()) -class TestUpdateAuthorizedKeys(test_helpers.TestCase): +class TestUpdateAuthorizedKeys(test_helpers.CiTestCase): def test_new_keys_replace(self): """new entries with the same base64 should replace old.""" @@ -168,7 +169,7 @@ class TestUpdateAuthorizedKeys(test_helpers.TestCase): self.assertEqual(expected, found) -class TestParseSSHConfig(test_helpers.TestCase): +class TestParseSSHConfig(test_helpers.CiTestCase): def setUp(self): self.load_file_patch = patch('cloudinit.ssh_util.util.load_file') @@ -235,4 +236,94 @@ class TestParseSSHConfig(test_helpers.TestCase): self.assertEqual('foo', ret[0].key) self.assertEqual('bar', ret[0].value) + +class TestUpdateSshConfigLines(test_helpers.CiTestCase): + """Test the update_ssh_config_lines method.""" + exlines = [ + "#PasswordAuthentication yes", + "UsePAM yes", + "# Comment line", + "AcceptEnv LANG LC_*", + "X11Forwarding no", + ] + pwauth = "PasswordAuthentication" + + def check_line(self, line, opt, val): + self.assertEqual(line.key, opt.lower()) + self.assertEqual(line.value, val) + self.assertIn(opt, str(line)) + self.assertIn(val, str(line)) + + def test_new_option_added(self): + """A single update of non-existing option.""" + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, {'MyKey': 'MyVal'}) + self.assertEqual(['MyKey'], result) + self.check_line(lines[-1], "MyKey", "MyVal") + + def test_commented_out_not_updated_but_appended(self): + """Implementation does not un-comment and update lines.""" + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, {self.pwauth: "no"}) + self.assertEqual([self.pwauth], result) + self.check_line(lines[-1], self.pwauth, "no") + + def test_single_option_updated(self): + """A single update should have change made and line updated.""" + opt, val = ("UsePAM", "no") + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, {opt: val}) + self.assertEqual([opt], result) + self.check_line(lines[1], opt, val) + + def test_multiple_updates_with_add(self): + """Verify multiple updates some added some changed, some not.""" + updates = {"UsePAM": "no", "X11Forwarding": "no", "NewOpt": "newval", + "AcceptEnv": "LANG ADD LC_*"} + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, updates) + self.assertEqual(set(["UsePAM", "NewOpt", "AcceptEnv"]), set(result)) + self.check_line(lines[3], "AcceptEnv", updates["AcceptEnv"]) + + def test_return_empty_if_no_changes(self): + """If there are no changes, then return should be empty list.""" + updates = {"UsePAM": "yes"} + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, updates) + self.assertEqual([], result) + self.assertEqual(self.exlines, [str(l) for l in lines]) + + def test_keycase_not_modified(self): + """Original case of key should not be changed on update. + This behavior is to keep original config as much intact as can be.""" + updates = {"usepam": "no"} + lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) + result = ssh_util.update_ssh_config_lines(lines, updates) + self.assertEqual(["usepam"], result) + self.assertEqual("UsePAM no", str(lines[1])) + + +class TestUpdateSshConfig(test_helpers.CiTestCase): + cfgdata = '\n'.join(["#Option val", "MyKey ORIG_VAL", ""]) + + def test_modified(self): + mycfg = self.tmp_path("ssh_config_1") + util.write_file(mycfg, self.cfgdata) + ret = ssh_util.update_ssh_config({"MyKey": "NEW_VAL"}, mycfg) + self.assertTrue(ret) + found = util.load_file(mycfg) + self.assertEqual(self.cfgdata.replace("ORIG_VAL", "NEW_VAL"), found) + # assert there is a newline at end of file (LP: #1677205) + self.assertEqual('\n', found[-1]) + + def test_not_modified(self): + mycfg = self.tmp_path("ssh_config_2") + util.write_file(mycfg, self.cfgdata) + with patch("cloudinit.ssh_util.util.write_file") as m_write_file: + ret = ssh_util.update_ssh_config({"MyKey": "ORIG_VAL"}, mycfg) + self.assertFalse(ret) + self.assertEqual(self.cfgdata, util.load_file(mycfg)) + m_write_file.assert_not_called() + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 53154d33..c36e6eb0 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -10,6 +10,7 @@ from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater +from cloudinit.util import load_file, write_file try: import Cheetah @@ -19,7 +20,20 @@ except ImportError: HAS_CHEETAH = False -class TestTemplates(test_helpers.TestCase): +class TestTemplates(test_helpers.CiTestCase): + + with_logs = True + + jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n' + jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8') + + @staticmethod + def add_header(renderer, data): + """Return text (py2 unicode/py3 str) with template header.""" + if isinstance(data, bytes): + data = data.decode('utf-8') + return "## template: %s\n" % renderer + data + def test_render_basic(self): in_data = textwrap.dedent(""" ${b} @@ -39,12 +53,12 @@ class TestTemplates(test_helpers.TestCase): def test_detection(self): blob = "## template:cheetah" - (template_type, renderer, contents) = templater.detect_template(blob) + (template_type, _renderer, contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) self.assertEqual("", contents.strip()) blob = "blahblah $blah" - (template_type, renderer, contents) = templater.detect_template(blob) + (template_type, _renderer, _contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) self.assertEqual(blob, contents) @@ -106,4 +120,52 @@ $a,$b''' 'codename': codename}) self.assertEqual(ex_data, out_data) + def test_jinja_nonascii_render_to_string(self): + """Test jinja render_to_string with non-ascii content.""" + self.assertEqual( + templater.render_string( + self.add_header("jinja", self.jinja_utf8), {"name": "bob"}), + self.jinja_utf8_rbob) + + def test_jinja_nonascii_render_undefined_variables_to_default_py3(self): + """Test py3 jinja render_to_string with undefined variable default.""" + self.assertEqual( + templater.render_string( + self.add_header("jinja", self.jinja_utf8), {}), + self.jinja_utf8_rbob.replace('bob', 'CI_MISSING_JINJA_VAR/name')) + + def test_jinja_nonascii_render_to_file(self): + """Test jinja render_to_file of a filename with non-ascii content.""" + tmpl_fn = self.tmp_path("j-render-to-file.template") + out_fn = self.tmp_path("j-render-to-file.out") + write_file(filename=tmpl_fn, omode="wb", + content=self.add_header( + "jinja", self.jinja_utf8).encode('utf-8')) + templater.render_to_file(tmpl_fn, out_fn, {"name": "bob"}) + result = load_file(out_fn, decode=False).decode('utf-8') + self.assertEqual(result, self.jinja_utf8_rbob) + + def test_jinja_nonascii_render_from_file(self): + """Test jinja render_from_file with non-ascii content.""" + tmpl_fn = self.tmp_path("j-render-from-file.template") + write_file(tmpl_fn, omode="wb", + content=self.add_header( + "jinja", self.jinja_utf8).encode('utf-8')) + result = templater.render_from_file(tmpl_fn, {"name": "bob"}) + self.assertEqual(result, self.jinja_utf8_rbob) + + @test_helpers.skipIfJinja() + def test_jinja_warns_on_missing_dep_and_uses_basic_renderer(self): + """Test jinja render_from_file will fallback to basic renderer.""" + tmpl_fn = self.tmp_path("j-render-from-file.template") + write_file(tmpl_fn, omode="wb", + content=self.add_header( + "jinja", self.jinja_utf8).encode('utf-8')) + result = templater.render_from_file(tmpl_fn, {"name": "bob"}) + self.assertEqual(result, self.jinja_utf8.decode()) + self.assertIn( + 'WARNING: Jinja not available as the selected renderer for desired' + ' template, reverting to the basic renderer.', + self.logs.getvalue()) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 50101906..5a14479a 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -4,6 +4,7 @@ from __future__ import print_function import logging import os +import re import shutil import stat import tempfile @@ -23,6 +24,7 @@ except ImportError: BASH = util.which('bash') +BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' class FakeSelinux(object): @@ -265,26 +267,49 @@ class TestGetCmdline(helpers.TestCase): self.assertEqual("abcd 123", ret) -class TestLoadYaml(helpers.TestCase): +class TestLoadYaml(helpers.CiTestCase): mydefault = "7b03a8ebace993d806255121073fed52" + with_logs = True def test_simple(self): mydata = {'1': "one", '2': "two"} self.assertEqual(util.load_yaml(yaml.dump(mydata)), mydata) def test_nonallowed_returns_default(self): + '''Any unallowed types result in returning default; log the issue.''' # for now, anything not in the allowed list just returns the default. myyaml = yaml.dump({'1': "one"}) self.assertEqual(util.load_yaml(blob=myyaml, default=self.mydefault, allowed=(str,)), self.mydefault) - - def test_bogus_returns_default(self): + regex = re.compile( + r'Yaml load allows \(<(class|type) \'str\'>,\) root types, but' + r' got dict') + self.assertTrue(regex.search(self.logs.getvalue()), + msg='Missing expected yaml load error') + + def test_bogus_scan_error_returns_default(self): + '''On Yaml scan error, load_yaml returns the default and logs issue.''' badyaml = "1\n 2:" self.assertEqual(util.load_yaml(blob=badyaml, default=self.mydefault), self.mydefault) + self.assertIn( + 'Failed loading yaml blob. Invalid format at line 2 column 3:' + ' "mapping values are not allowed here', + self.logs.getvalue()) + + def test_bogus_parse_error_returns_default(self): + '''On Yaml parse error, load_yaml returns default and logs issue.''' + badyaml = "{}}" + self.assertEqual(util.load_yaml(blob=badyaml, + default=self.mydefault), + self.mydefault) + self.assertIn( + 'Failed loading yaml blob. Invalid format at line 1 column 3:' + " \"expected \'<document start>\', but found \'}\'", + self.logs.getvalue()) def test_unsafe_types(self): # should not load complex types @@ -325,7 +350,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): def test_precise_ext4_root(self): - lines = self.readResource('mountinfo_precise_ext4.txt').splitlines() + lines = helpers.readResource('mountinfo_precise_ext4.txt').splitlines() expected = ('/dev/mapper/vg0-root', 'ext4', '/') self.assertEqual(expected, util.parse_mount_info('/', lines)) @@ -347,7 +372,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertEqual(expected, util.parse_mount_info('/run/lock', lines)) def test_raring_btrfs_root(self): - lines = self.readResource('mountinfo_raring_btrfs.txt').splitlines() + lines = helpers.readResource('mountinfo_raring_btrfs.txt').splitlines() expected = ('/dev/vda1', 'btrfs', '/') self.assertEqual(expected, util.parse_mount_info('/', lines)) @@ -373,7 +398,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): m_os.path.exists.return_value = True # mock subp command from util.get_mount_info_fs_on_zpool zpool_output.return_value = ( - self.readResource('zpool_status_simple.txt'), '' + helpers.readResource('zpool_status_simple.txt'), '' ) # save function return values and do asserts ret = util.get_device_info_from_zpool('vmzroot') @@ -406,7 +431,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): m_os.path.exists.return_value = True # mock subp command from util.get_mount_info_fs_on_zpool zpool_output.return_value = ( - self.readResource('zpool_status_simple.txt'), 'error' + helpers.readResource('zpool_status_simple.txt'), 'error' ) # save function return values and do asserts ret = util.get_device_info_from_zpool('vmzroot') @@ -414,7 +439,8 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): @mock.patch('cloudinit.util.subp') def test_parse_mount_with_ext(self, mount_out): - mount_out.return_value = (self.readResource('mount_parse_ext.txt'), '') + mount_out.return_value = ( + helpers.readResource('mount_parse_ext.txt'), '') # this one is valid and exists in mount_parse_ext.txt ret = util.parse_mount('/var') self.assertEqual(('/dev/mapper/vg00-lv_var', 'ext4', '/var'), ret) @@ -430,7 +456,8 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): @mock.patch('cloudinit.util.subp') def test_parse_mount_with_zfs(self, mount_out): - mount_out.return_value = (self.readResource('mount_parse_zfs.txt'), '') + mount_out.return_value = ( + helpers.readResource('mount_parse_zfs.txt'), '') # this one is valid and exists in mount_parse_zfs.txt ret = util.parse_mount('/var') self.assertEqual(('vmzroot/ROOT/freebsd/var', 'zfs', '/var'), ret) @@ -442,6 +469,29 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertIsNone(ret) +class TestIsX86(helpers.CiTestCase): + + def test_is_x86_matches_x86_types(self): + """is_x86 returns True if CPU architecture matches.""" + matched_arches = ['x86_64', 'i386', 'i586', 'i686'] + for arch in matched_arches: + self.assertTrue( + util.is_x86(arch), 'Expected is_x86 for arch "%s"' % arch) + + def test_is_x86_unmatched_types(self): + """is_x86 returns Fale on non-intel x86 architectures.""" + unmatched_arches = ['ia64', '9000/800', 'arm64v71'] + for arch in unmatched_arches: + self.assertFalse( + util.is_x86(arch), 'Expected not is_x86 for arch "%s"' % arch) + + @mock.patch('cloudinit.util.os.uname') + def test_is_x86_calls_uname_for_architecture(self, m_uname): + """is_x86 returns True if platform from uname matches.""" + m_uname.return_value = [0, 1, 2, 3, 'x86_64'] + self.assertTrue(util.is_x86()) + + class TestReadDMIData(helpers.FilesystemMockingTestCase): def setUp(self): @@ -693,6 +743,8 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.CiTestCase): with_logs = True + allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, + BOGUS_COMMAND, sys.executable] stdin2err = [BASH, '-c', 'cat >&2'] stdin2out = ['cat'] @@ -700,7 +752,6 @@ class TestSubp(helpers.CiTestCase): utf8_valid = b'start \xc3\xa9 end' utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] - bogus_command = 'this-is-not-expected-to-be-a-program-name' def printf_cmd(self, *args): # bash's printf supports \xaa. So does /usr/bin/printf @@ -772,11 +823,11 @@ class TestSubp(helpers.CiTestCase): def test_subp_reads_env(self): with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): - out, err = util.subp(self.printenv + ['FOO'], capture=True) + out, _err = util.subp(self.printenv + ['FOO'], capture=True) self.assertEqual('FOO=BAR', out.splitlines()[0]) def test_subp_env_and_update_env(self): - out, err = util.subp( + out, _err = util.subp( self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, env={'FOO': 'BAR'}, update_env={'HOME': '/myhome', 'K2': 'V2'}) @@ -786,7 +837,7 @@ class TestSubp(helpers.CiTestCase): def test_subp_update_env(self): extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} with mock.patch.dict("os.environ", values=extra): - out, err = util.subp( + out, _err = util.subp( self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, update_env={'HOME': '/myhome', 'K2': 'V2'}) @@ -799,9 +850,18 @@ class TestSubp(helpers.CiTestCase): util.write_file(noshebang, 'true\n') os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) - self.assertRaisesRegex(util.ProcessExecutionError, - 'Missing #! in script\?', - util.subp, (noshebang,)) + with self.allow_subp([noshebang]): + self.assertRaisesRegex(util.ProcessExecutionError, + r'Missing #! in script\?', + util.subp, (noshebang,)) + + def test_subp_combined_stderr_stdout(self): + """Providing combine_capture as True redirects stderr to stdout.""" + data = b'hello world' + (out, err) = util.subp(self.stdin2err, capture=True, + combine_capture=True, decode=False, data=data) + self.assertEqual(b'', err) + self.assertEqual(data, out) def test_returns_none_if_no_capture(self): (out, err) = util.subp(self.stdin2out, data=b'', capture=False) @@ -811,14 +871,14 @@ class TestSubp(helpers.CiTestCase): def test_exception_has_out_err_are_bytes_if_decode_false(self): """Raised exc should have stderr, stdout as bytes if no decode.""" with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([self.bogus_command], decode=False) + util.subp([BOGUS_COMMAND], decode=False) self.assertTrue(isinstance(cm.exception.stdout, bytes)) self.assertTrue(isinstance(cm.exception.stderr, bytes)) def test_exception_has_out_err_are_bytes_if_decode_true(self): """Raised exc should have stderr, stdout as string if no decode.""" with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([self.bogus_command], decode=True) + util.subp([BOGUS_COMMAND], decode=True) self.assertTrue(isinstance(cm.exception.stdout, six.string_types)) self.assertTrue(isinstance(cm.exception.stderr, six.string_types)) @@ -868,10 +928,10 @@ class TestSubp(helpers.CiTestCase): logs.append(log) with self.assertRaises(util.ProcessExecutionError): - util.subp([self.bogus_command], status_cb=status_cb) + util.subp([BOGUS_COMMAND], status_cb=status_cb) expected = [ - 'Begin run command: {cmd}\n'.format(cmd=self.bogus_command), + 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), 'ERROR: End run command: invalid command provided\n'] self.assertEqual(expected, logs) @@ -883,13 +943,13 @@ class TestSubp(helpers.CiTestCase): logs.append(log) with self.assertRaises(util.ProcessExecutionError): - util.subp(['ls', '/I/dont/exist'], status_cb=status_cb) - util.subp(['ls'], status_cb=status_cb) + util.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) + util.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) expected = [ - 'Begin run command: ls /I/dont/exist\n', + 'Begin run command: %s -c exit 2\n' % BASH, 'ERROR: End run command: exit(2)\n', - 'Begin run command: ls\n', + 'Begin run command: %s -c exit 0\n' % BASH, 'End run command: exit(0)\n'] self.assertEqual(expected, logs) @@ -1055,4 +1115,60 @@ class TestLoadShellContent(helpers.TestCase): '']))) +class TestGetProcEnv(helpers.TestCase): + """test get_proc_env.""" + null = b'\x00' + simple1 = b'HOME=/' + simple2 = b'PATH=/bin:/sbin' + bootflag = b'BOOTABLE_FLAG=\x80' # from LP: #1775371 + mixed = b'MIXED=' + b'ab\xccde' + + def _val_decoded(self, blob, encoding='utf-8', errors='replace'): + # return the value portion of key=val decoded. + return blob.split(b'=', 1)[1].decode(encoding, errors) + + @mock.patch("cloudinit.util.load_file") + def test_non_utf8_in_environment(self, m_load_file): + """env may have non utf-8 decodable content.""" + content = self.null.join( + (self.bootflag, self.simple1, self.simple2, self.mixed)) + m_load_file.return_value = content + + self.assertEqual( + {'BOOTABLE_FLAG': self._val_decoded(self.bootflag), + 'HOME': '/', 'PATH': '/bin:/sbin', + 'MIXED': self._val_decoded(self.mixed)}, + util.get_proc_env(1)) + self.assertEqual(1, m_load_file.call_count) + + @mock.patch("cloudinit.util.load_file") + def test_encoding_none_returns_bytes(self, m_load_file): + """encoding none returns bytes.""" + lines = (self.bootflag, self.simple1, self.simple2, self.mixed) + content = self.null.join(lines) + m_load_file.return_value = content + + self.assertEqual( + dict([t.split(b'=') for t in lines]), + util.get_proc_env(1, encoding=None)) + self.assertEqual(1, m_load_file.call_count) + + @mock.patch("cloudinit.util.load_file") + def test_all_utf8_encoded(self, m_load_file): + """common path where only utf-8 decodable content.""" + content = self.null.join((self.simple1, self.simple2)) + m_load_file.return_value = content + self.assertEqual( + {'HOME': '/', 'PATH': '/bin:/sbin'}, + util.get_proc_env(1)) + self.assertEqual(1, m_load_file.call_count) + + @mock.patch("cloudinit.util.load_file") + def test_non_existing_file_returns_empty_dict(self, m_load_file): + """as implemented, a non-existing pid returns empty dict. + This is how it was originally implemented.""" + m_load_file.side_effect = OSError("File does not exist.") + self.assertEqual({}, util.get_proc_env(1)) + self.assertEqual(1, m_load_file.call_count) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py deleted file mode 100644 index d012f69d..00000000 --- a/tests/unittests/test_version.py +++ /dev/null @@ -1,14 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit.tests.helpers import CiTestCase -from cloudinit import version - - -class TestExportsFeatures(CiTestCase): - def test_has_network_config_v1(self): - self.assertIn('NETWORK_CONFIG_V1', version.FEATURES) - - def test_has_network_config_v2(self): - self.assertIn('NETWORK_CONFIG_V2', version.FEATURES) - -# vi: ts=4 expandtab diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 036f6879..602dedb0 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -2,11 +2,15 @@ # Copyright (C) 2016 VMware INC. # # Author: Sankar Tanguturi <stanguturi@vmware.com> +# Pengpeng Sun <pengpengs@vmware.com> # # This file is part of cloud-init. See LICENSE file for license information. import logging +import os import sys +import tempfile +import textwrap from cloudinit.sources.DataSourceOVF import get_network_config_from_conf from cloudinit.sources.DataSourceOVF import read_vmware_imc @@ -343,4 +347,115 @@ class TestVmwareConfigFile(CiTestCase): conf = Config(cf) self.assertEqual("test-script", conf.custom_script_name) + +class TestVmwareNetConfig(CiTestCase): + """Test conversion of vmware config to cloud-init config.""" + + def _get_NicConfigurator(self, text): + fp = None + try: + with tempfile.NamedTemporaryFile(mode="w", dir=self.tmp_dir(), + delete=False) as fp: + fp.write(text) + fp.close() + cfg = Config(ConfigFile(fp.name)) + return NicConfigurator(cfg.nics, use_system_devices=False) + finally: + if fp: + os.unlink(fp.name) + + def test_non_primary_nic_without_gateway(self): + """A non primary nic set is not required to have a gateway.""" + config = textwrap.dedent("""\ + [NETWORK] + NETWORKING = yes + BOOTPROTO = dhcp + HOSTNAME = myhost1 + DOMAINNAME = eng.vmware.com + + [NIC-CONFIG] + NICS = NIC1 + + [NIC1] + MACADDR = 00:50:56:a6:8c:08 + ONBOOT = yes + IPv4_MODE = BACKWARDS_COMPATIBLE + BOOTPROTO = static + IPADDR = 10.20.87.154 + NETMASK = 255.255.252.0 + """) + nc = self._get_NicConfigurator(config) + self.assertEqual( + [{'type': 'physical', 'name': 'NIC1', + 'mac_address': '00:50:56:a6:8c:08', + 'subnets': [ + {'control': 'auto', 'type': 'static', + 'address': '10.20.87.154', 'netmask': '255.255.252.0'}]}], + nc.generate()) + + def test_non_primary_nic_with_gateway(self): + """A non primary nic set can have a gateway.""" + config = textwrap.dedent("""\ + [NETWORK] + NETWORKING = yes + BOOTPROTO = dhcp + HOSTNAME = myhost1 + DOMAINNAME = eng.vmware.com + + [NIC-CONFIG] + NICS = NIC1 + + [NIC1] + MACADDR = 00:50:56:a6:8c:08 + ONBOOT = yes + IPv4_MODE = BACKWARDS_COMPATIBLE + BOOTPROTO = static + IPADDR = 10.20.87.154 + NETMASK = 255.255.252.0 + GATEWAY = 10.20.87.253 + """) + nc = self._get_NicConfigurator(config) + self.assertEqual( + [{'type': 'physical', 'name': 'NIC1', + 'mac_address': '00:50:56:a6:8c:08', + 'subnets': [ + {'control': 'auto', 'type': 'static', + 'address': '10.20.87.154', 'netmask': '255.255.252.0'}]}, + {'type': 'route', 'destination': '10.20.84.0/22', + 'gateway': '10.20.87.253', 'metric': 10000}], + nc.generate()) + + def test_a_primary_nic_with_gateway(self): + """A primary nic set can have a gateway.""" + config = textwrap.dedent("""\ + [NETWORK] + NETWORKING = yes + BOOTPROTO = dhcp + HOSTNAME = myhost1 + DOMAINNAME = eng.vmware.com + + [NIC-CONFIG] + NICS = NIC1 + + [NIC1] + MACADDR = 00:50:56:a6:8c:08 + ONBOOT = yes + IPv4_MODE = BACKWARDS_COMPATIBLE + BOOTPROTO = static + IPADDR = 10.20.87.154 + NETMASK = 255.255.252.0 + PRIMARY = true + GATEWAY = 10.20.87.253 + """) + nc = self._get_NicConfigurator(config) + self.assertEqual( + [{'type': 'physical', 'name': 'NIC1', + 'mac_address': '00:50:56:a6:8c:08', + 'subnets': [ + {'control': 'auto', 'type': 'static', + 'address': '10.20.87.154', 'netmask': '255.255.252.0', + 'gateway': '10.20.87.253'}]}], + nc.generate()) + + # vi: ts=4 expandtab |