From 47016791ca5e97d80e45d3f100bc4e5d0b88627d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Dec 2017 17:05:29 -0500 Subject: tests: consolidate platforms into specific dirs This groups up each test platform into its own directory rather than having files spread between four different directories for one platform. Platforms tend to be worked on one at a time and so having the platforms together makes more sense than apart. --- tests/cloud_tests/bddeb.py | 3 +- tests/cloud_tests/collect.py | 8 +- tests/cloud_tests/images/__init__.py | 10 -- tests/cloud_tests/images/base.py | 56 ------ tests/cloud_tests/images/lxd.py | 194 --------------------- tests/cloud_tests/images/nocloudkvm.py | 90 ---------- tests/cloud_tests/instances/__init__.py | 10 -- tests/cloud_tests/instances/base.py | 77 -------- tests/cloud_tests/instances/lxd.py | 157 ----------------- tests/cloud_tests/instances/nocloudkvm.py | 179 ------------------- tests/cloud_tests/platforms/__init__.py | 20 ++- tests/cloud_tests/platforms/base.py | 27 --- tests/cloud_tests/platforms/images.py | 56 ++++++ tests/cloud_tests/platforms/instances.py | 77 ++++++++ tests/cloud_tests/platforms/lxd.py | 108 ------------ tests/cloud_tests/platforms/lxd/image.py | 193 ++++++++++++++++++++ tests/cloud_tests/platforms/lxd/instance.py | 157 +++++++++++++++++ tests/cloud_tests/platforms/lxd/platform.py | 108 ++++++++++++ tests/cloud_tests/platforms/lxd/snapshot.py | 53 ++++++ tests/cloud_tests/platforms/nocloudkvm.py | 91 ---------- tests/cloud_tests/platforms/nocloudkvm/image.py | 89 ++++++++++ tests/cloud_tests/platforms/nocloudkvm/instance.py | 179 +++++++++++++++++++ tests/cloud_tests/platforms/nocloudkvm/platform.py | 89 ++++++++++ tests/cloud_tests/platforms/nocloudkvm/snapshot.py | 79 +++++++++ tests/cloud_tests/platforms/platforms.py | 27 +++ tests/cloud_tests/platforms/snapshots.py | 45 +++++ tests/cloud_tests/snapshots/__init__.py | 10 -- tests/cloud_tests/snapshots/base.py | 45 ----- tests/cloud_tests/snapshots/lxd.py | 53 ------ tests/cloud_tests/snapshots/nocloudkvm.py | 79 --------- 30 files changed, 1176 insertions(+), 1193 deletions(-) delete mode 100644 tests/cloud_tests/images/__init__.py delete mode 100644 tests/cloud_tests/images/base.py delete mode 100644 tests/cloud_tests/images/lxd.py delete mode 100644 tests/cloud_tests/images/nocloudkvm.py delete mode 100644 tests/cloud_tests/instances/__init__.py delete mode 100644 tests/cloud_tests/instances/base.py delete mode 100644 tests/cloud_tests/instances/lxd.py delete mode 100644 tests/cloud_tests/instances/nocloudkvm.py delete mode 100644 tests/cloud_tests/platforms/base.py create mode 100644 tests/cloud_tests/platforms/images.py create mode 100644 tests/cloud_tests/platforms/instances.py delete mode 100644 tests/cloud_tests/platforms/lxd.py create mode 100644 tests/cloud_tests/platforms/lxd/image.py create mode 100644 tests/cloud_tests/platforms/lxd/instance.py create mode 100644 tests/cloud_tests/platforms/lxd/platform.py create mode 100644 tests/cloud_tests/platforms/lxd/snapshot.py delete mode 100644 tests/cloud_tests/platforms/nocloudkvm.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm/image.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm/instance.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm/platform.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm/snapshot.py create mode 100644 tests/cloud_tests/platforms/platforms.py create mode 100644 tests/cloud_tests/platforms/snapshots.py delete mode 100644 tests/cloud_tests/snapshots/__init__.py delete mode 100644 tests/cloud_tests/snapshots/base.py delete mode 100644 tests/cloud_tests/snapshots/lxd.py delete mode 100644 tests/cloud_tests/snapshots/nocloudkvm.py (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index fba8a0c7..c259dfea 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -8,7 +8,8 @@ import tempfile from cloudinit import util as c_util from tests.cloud_tests import (config, LOG) -from tests.cloud_tests import (platforms, images, snapshots, instances) +from tests.cloud_tests.platforms import (platforms, images, snapshots, + instances) from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) pre_reqs = ['devscripts', 'equivs', 'git', 'tar'] diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 71ee7645..db5ee99f 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -8,7 +8,7 @@ import os from cloudinit import util as c_util from tests.cloud_tests import (config, LOG, setup_image, util) from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) -from tests.cloud_tests import (platforms, images, snapshots, instances) +from tests.cloud_tests import platforms def collect_script(instance, base_dir, script, script_name): @@ -77,7 +77,7 @@ def collect_test_data(args, snapshot, os_name, test_name): # create test instance component = PlatformComponent( - partial(instances.get_instance, snapshot, user_data, + partial(platforms.get_instance, snapshot, user_data, block=True, start=False, use_desc=test_name)) LOG.info('collecting test data for test: %s', test_name) @@ -108,7 +108,7 @@ def collect_snapshot(args, image, os_name): """ res = ({}, 1) - component = PlatformComponent(partial(snapshots.get_snapshot, image)) + component = PlatformComponent(partial(platforms.get_snapshot, image)) LOG.debug('creating snapshot for %s', os_name) with component as snapshot: @@ -136,7 +136,7 @@ def collect_image(args, platform, os_name): feature_overrides=args.feature_override) LOG.debug('os config: %s', os_config) component = PlatformComponent( - partial(images.get_image, platform, os_config)) + partial(platforms.get_image, platform, os_config)) LOG.info('acquiring image for os: %s', os_name) with component as image: diff --git a/tests/cloud_tests/images/__init__.py b/tests/cloud_tests/images/__init__.py deleted file mode 100644 index 106c59f3..00000000 --- a/tests/cloud_tests/images/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Main init.""" - - -def get_image(platform, config): - """Get image from platform object using os_name.""" - return platform.get_image(config) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/images/base.py b/tests/cloud_tests/images/base.py deleted file mode 100644 index d503108a..00000000 --- a/tests/cloud_tests/images/base.py +++ /dev/null @@ -1,56 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base class for images.""" - -from ..util import TargetBase - - -class Image(TargetBase): - """Base class for images.""" - - platform_name = None - - def __init__(self, platform, config): - """Set up image. - - @param platform: platform object - @param config: image configuration - """ - self.platform = platform - self.config = config - - def __str__(self): - """A brief description of the image.""" - return '-'.join((self.properties['os'], self.properties['release'])) - - @property - def properties(self): - """{} containing: 'arch', 'os', 'version', 'release'.""" - raise NotImplementedError - - @property - def features(self): - """Feature flags supported by this image. - - @return_value: list of feature names - """ - return [k for k, v in self.config.get('features', {}).items() if v] - - @property - def setup_overrides(self): - """Setup options that need to be overridden for the image. - - @return_value: dictionary to update args with - """ - # NOTE: more sophisticated options may be requied at some point - return self.config.get('setup_overrides', {}) - - def snapshot(self): - """Create snapshot of image, block until done.""" - raise NotImplementedError - - def destroy(self): - """Clean up data associated with image.""" - pass - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/images/lxd.py b/tests/cloud_tests/images/lxd.py deleted file mode 100644 index 5caeba41..00000000 --- a/tests/cloud_tests/images/lxd.py +++ /dev/null @@ -1,194 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""LXD Image Base Class.""" - -import os -import shutil -import tempfile - -from cloudinit import util as c_util -from tests.cloud_tests.images import base -from tests.cloud_tests.snapshots import lxd as lxd_snapshot -from tests.cloud_tests import util - - -class LXDImage(base.Image): - """LXD backed image.""" - - platform_name = "lxd" - - def __init__(self, platform, config, pylxd_image): - """Set up image. - - @param platform: platform object - @param config: image configuration - """ - self.modified = False - self._img_instance = None - self._pylxd_image = None - self.pylxd_image = pylxd_image - super(LXDImage, self).__init__(platform, config) - - @property - def pylxd_image(self): - """Property function.""" - if self._pylxd_image: - self._pylxd_image.sync() - return self._pylxd_image - - @pylxd_image.setter - def pylxd_image(self, pylxd_image): - if self._img_instance: - self._instance.destroy() - self._img_instance = None - if (self._pylxd_image and - (self._pylxd_image is not pylxd_image) and - (not self.config.get('cache_base_image') or self.modified)): - self._pylxd_image.delete(wait=True) - self.modified = False - self._pylxd_image = pylxd_image - - @property - def _instance(self): - """Internal use only, returns a instance - - This starts an lxc instance from the image, so it is "dirty". - Better would be some way to modify this "at rest". - lxc-pstart would be an option.""" - if not self._img_instance: - self._img_instance = self.platform.launch_container( - self.properties, self.config, self.features, - use_desc='image-modification', image_desc=str(self), - image=self.pylxd_image.fingerprint) - self._img_instance.start() - return self._img_instance - - @property - def properties(self): - """{} containing: 'arch', 'os', 'version', 'release'.""" - properties = self.pylxd_image.properties - return { - 'arch': properties.get('architecture'), - 'os': properties.get('os'), - 'version': properties.get('version'), - 'release': properties.get('release'), - } - - def export_image(self, output_dir): - """Export image from lxd image store to (split) tarball on disk. - - @param output_dir: dir to store tarballs in - @return_value: tuple of path to metadata tarball and rootfs tarball - """ - # pylxd's image export feature doesn't do split exports, so use cmdline - c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, - output_dir], capture=True) - tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] - metadata = os.path.join( - output_dir, next(p for p in tarballs if p.startswith('meta-'))) - rootfs = os.path.join( - output_dir, next(p for p in tarballs if not p.startswith('meta-'))) - return (metadata, rootfs) - - def import_image(self, metadata, rootfs): - """Import image to lxd image store from (split) tarball on disk. - - Note, this will replace and delete the current pylxd_image - - @param metadata: metadata tarball - @param rootfs: rootfs tarball - @return_value: imported image fingerprint - """ - alias = util.gen_instance_name( - image_desc=str(self), use_desc='update-metadata') - c_util.subp(['lxc', 'image', 'import', metadata, rootfs, - '--alias', alias], capture=True) - self.pylxd_image = self.platform.query_image_by_alias(alias) - return self.pylxd_image.fingerprint - - def update_templates(self, template_config, template_data): - """Update the image's template configuration. - - Note, this will replace and delete the current pylxd_image - - @param template_config: config overrides for template metadata - @param template_data: template data to place into templates/ - """ - # set up tmp files - export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') - extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') - new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') - metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') - template_dir = os.path.join(extract_dir, 'templates') - - try: - # extract old data - (metadata, rootfs) = self.export_image(export_dir) - shutil.unpack_archive(metadata, extract_dir) - - # update metadata - metadata = c_util.read_conf(metadata_yaml) - templates = metadata.get('templates', {}) - templates.update(template_config) - metadata['templates'] = templates - util.yaml_dump(metadata, metadata_yaml) - - # write out template files - for name, content in template_data.items(): - path = os.path.join(template_dir, name) - c_util.write_file(path, content) - - # store new data, mark new image as modified - util.flat_tar(new_metadata, extract_dir) - self.import_image(new_metadata, rootfs) - self.modified = True - - finally: - # remove tmpfiles - shutil.rmtree(export_dir) - shutil.rmtree(extract_dir) - - def _execute(self, *args, **kwargs): - """Execute command in image, modifying image.""" - return self._instance._execute(*args, **kwargs) - - def push_file(self, local_path, remote_path): - """Copy file at 'local_path' to instance at 'remote_path'.""" - return self._instance.push_file(local_path, remote_path) - - def run_script(self, *args, **kwargs): - """Run script in image, modifying image. - - @return_value: script output - """ - return self._instance.run_script(*args, **kwargs) - - def snapshot(self): - """Create snapshot of image, block until done.""" - # get empty user data to pass in to instance - # if overrides for user data provided, use them - empty_userdata = util.update_user_data( - {}, self.config.get('user_data_overrides', {})) - conf = {'user.user-data': empty_userdata} - # clone current instance - instance = self.platform.launch_container( - self.properties, self.config, self.features, - container=self._instance.name, image_desc=str(self), - use_desc='snapshot', container_config=conf) - # wait for cloud-init before boot_clean_script is run to ensure - # /var/lib/cloud is removed cleanly - instance.start(wait=True, wait_for_cloud_init=True) - if self.config.get('boot_clean_script'): - instance.run_script(self.config.get('boot_clean_script')) - # freeze current instance and return snapshot - instance.freeze() - return lxd_snapshot.LXDSnapshot( - self.platform, self.properties, self.config, - self.features, instance) - - def destroy(self): - """Clean up data associated with image.""" - self.pylxd_image = None - super(LXDImage, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/images/nocloudkvm.py b/tests/cloud_tests/images/nocloudkvm.py deleted file mode 100644 index 8678b07f..00000000 --- a/tests/cloud_tests/images/nocloudkvm.py +++ /dev/null @@ -1,90 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""NoCloud KVM Image Base Class.""" - -from cloudinit import util as c_util - -import os -import shutil -import tempfile - -from tests.cloud_tests.images import base -from tests.cloud_tests.snapshots import nocloudkvm as nocloud_kvm_snapshot - - -class NoCloudKVMImage(base.Image): - """NoCloud KVM backed image.""" - - platform_name = "nocloud-kvm" - - def __init__(self, platform, config, orig_img_path): - """Set up image. - - @param platform: platform object - @param config: image configuration - @param img_path: path to the image - """ - self.modified = False - self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage') - self._orig_img_path = orig_img_path - self._img_path = os.path.join(self._workd, - os.path.basename(self._orig_img_path)) - - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', orig_img_path, self._img_path]) - - super(NoCloudKVMImage, self).__init__(platform, config) - - @property - def properties(self): - """Dictionary containing: 'arch', 'os', 'version', 'release'.""" - return { - 'arch': self.config['arch'], - 'os': self.config['family'], - 'release': self.config['release'], - 'version': self.config['version'], - } - - def _execute(self, command, stdin=None, env=None): - """Execute command in image, modifying image.""" - return self.mount_image_callback(command, stdin=stdin, env=env) - - def mount_image_callback(self, command, stdin=None, env=None): - """Run mount-image-callback.""" - - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts', - '--system-resolvconf', self._img_path, - '--', 'chroot', '_MOUNTPOINT_'] - try: - out, err = c_util.subp(mic_chroot + env_args + list(command), - data=stdin, decode=False) - return (out, err, 0) - except c_util.ProcessExecutionError as e: - return (e.stdout, e.stderr, e.exit_code) - - def snapshot(self): - """Create snapshot of image, block until done.""" - if not self._img_path: - raise RuntimeError() - - return nocloud_kvm_snapshot.NoCloudKVMSnapshot( - self.platform, self.properties, self.config, - self.features, self._img_path) - - def destroy(self): - """Unset path to signal image is no longer used. - - The removal of the images and all other items is handled by the - framework. In some cases we want to keep the images, so let the - framework decide whether to keep or destroy everything. - """ - self._img_path = None - shutil.rmtree(self._workd) - - super(NoCloudKVMImage, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/__init__.py b/tests/cloud_tests/instances/__init__.py deleted file mode 100644 index fc2e9cbc..00000000 --- a/tests/cloud_tests/instances/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Main init.""" - - -def get_instance(snapshot, *args, **kwargs): - """Get instance from snapshot.""" - return snapshot.launch(*args, **kwargs) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py deleted file mode 100644 index 8c59d62c..00000000 --- a/tests/cloud_tests/instances/base.py +++ /dev/null @@ -1,77 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base instance.""" - -from ..util import TargetBase - - -class Instance(TargetBase): - """Base instance object.""" - - platform_name = None - - def __init__(self, platform, name, properties, config, features): - """Set up instance. - - @param platform: platform object - @param name: hostname of instance - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.platform = platform - self.name = name - self.properties = properties - self.config = config - self.features = features - self._tmp_count = 0 - - def console_log(self): - """Instance console. - - @return_value: bytes of this instance’s console - """ - raise NotImplementedError - - def reboot(self, wait=True): - """Reboot instance.""" - raise NotImplementedError - - def shutdown(self, wait=True): - """Shutdown instance.""" - raise NotImplementedError - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - raise NotImplementedError - - def destroy(self): - """Clean up instance.""" - pass - - def _wait_for_system(self, wait_for_cloud_init): - """Wait until system has fully booted and cloud-init has finished. - - @param wait_time: maximum time to wait - @return_value: None, may raise OSError if wait_time exceeded - """ - def clean_test(test): - """Clean formatting for system ready test testcase.""" - return ' '.join(l for l in test.strip().splitlines() - if not l.lstrip().startswith('#')) - - time = self.config['boot_timeout'] - tests = [self.config['system_ready_script']] - if wait_for_cloud_init: - tests.append(self.config['cloud_init_ready_script']) - - formatted_tests = ' && '.join(clean_test(t) for t in tests) - cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' - 'exit 0; sleep 1; done; exit 1').format(time=time, - test=formatted_tests) - - if self.execute(cmd, rcs=(0, 1))[-1] != 0: - raise OSError('timeout: after {}s system not started'.format(time)) - - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py deleted file mode 100644 index 3b035d86..00000000 --- a/tests/cloud_tests/instances/lxd.py +++ /dev/null @@ -1,157 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD instance.""" - -from . import base - -import os -import shutil -from tempfile import mkdtemp - - -class LXDInstance(base.Instance): - """LXD container backed instance.""" - - platform_name = "lxd" - - def __init__(self, platform, name, properties, config, features, - pylxd_container): - """Set up instance. - - @param platform: platform object - @param name: hostname of instance - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self._pylxd_container = pylxd_container - super(LXDInstance, self).__init__( - platform, name, properties, config, features) - self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) - self._setup_console_log() - - @property - def pylxd_container(self): - """Property function.""" - self._pylxd_container.sync() - return self._pylxd_container - - def _setup_console_log(self): - logf = os.path.join(self.tmpd, "console.log") - - # doing this ensures we can read it. Otherwise it ends up root:root. - with open(logf, "w") as fp: - fp.write("# %s\n" % self.name) - - cfg = "lxc.console.logfile=%s" % logf - orig = self._pylxd_container.config.get('raw.lxc', "") - if orig: - orig += "\n" - self._pylxd_container.config['raw.lxc'] = orig + cfg - self._pylxd_container.save() - self._console_log_file = logf - - def _execute(self, command, stdin=None, env=None): - if env is None: - env = {} - - if stdin is not None: - # pylxd does not support input to execute. - # https://github.com/lxc/pylxd/issues/244 - # - # The solution here is write a tmp file in the container - # and then execute a shell that sets it standard in to - # be from that file, removes it, and calls the comand. - tmpf = self.tmpfile() - self.write_data(tmpf, stdin) - ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"' - command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] + - list(command)) - - # ensure instance is running and execute the command - self.start() - # execute returns a ContainerExecuteResult, named tuple - # (exit_code, stdout, stderr) - res = self.pylxd_container.execute(command, environment=env) - - # get out, exit and err from pylxd return - if not hasattr(res, 'exit_code'): - # pylxd 2.1.3 and earlier only return out and err, no exit - raise RuntimeError( - "No 'exit_code' in pylxd.container.execute return.\n" - "pylxd > 2.2 is required.") - - return res.stdout, res.stderr, res.exit_code - - def read_data(self, remote_path, decode=False): - """Read data from instance filesystem. - - @param remote_path: path in instance - @param decode: decode data before returning. - @return_value: content of remote_path as bytes if 'decode' is False, - and as string if 'decode' is True. - """ - data = self.pylxd_container.files.get(remote_path) - return data.decode() if decode else data - - def write_data(self, remote_path, data): - """Write data to instance filesystem. - - @param remote_path: path in instance - @param data: data to write in bytes - """ - self.pylxd_container.files.put(remote_path, data) - - def console_log(self): - """Console log. - - @return_value: bytes of this instance’s console - """ - if not os.path.exists(self._console_log_file): - raise NotImplementedError( - "Console log '%s' does not exist. If this is a remote " - "lxc, then this is really NotImplementedError. If it is " - "A local lxc, then this is a RuntimeError." - "https://github.com/lxc/lxd/issues/1129") - with open(self._console_log_file, "rb") as fp: - return fp.read() - - def reboot(self, wait=True): - """Reboot instance.""" - self.shutdown(wait=wait) - self.start(wait=wait) - - def shutdown(self, wait=True): - """Shutdown instance.""" - if self.pylxd_container.status != 'Stopped': - self.pylxd_container.stop(wait=wait) - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - if self.pylxd_container.status != 'Running': - self.pylxd_container.start(wait=wait) - if wait: - self._wait_for_system(wait_for_cloud_init) - - def freeze(self): - """Freeze instance.""" - if self.pylxd_container.status != 'Frozen': - self.pylxd_container.freeze(wait=True) - - def unfreeze(self): - """Unfreeze instance.""" - if self.pylxd_container.status == 'Frozen': - self.pylxd_container.unfreeze(wait=True) - - def destroy(self): - """Clean up instance.""" - self.unfreeze() - self.shutdown() - self.pylxd_container.delete(wait=True) - if self.platform.container_exists(self.name): - raise OSError('container {} was not properly removed' - .format(self.name)) - shutil.rmtree(self.tmpd) - super(LXDInstance, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/nocloudkvm.py b/tests/cloud_tests/instances/nocloudkvm.py deleted file mode 100644 index bc06a79e..00000000 --- a/tests/cloud_tests/instances/nocloudkvm.py +++ /dev/null @@ -1,179 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM instance.""" - -import os -import paramiko -import socket -import subprocess -import time - -from cloudinit import util as c_util -from tests.cloud_tests.instances import base -from tests.cloud_tests import util - -# This domain contains reverse lookups for hostnames that are used. -# The primary reason is so sudo will return quickly when it attempts -# to look up the hostname. i9n is just short for 'integration'. -# see also bug 1730744 for why we had to do this. -CI_DOMAIN = "i9n.cloud-init.io" - - -class NoCloudKVMInstance(base.Instance): - """NoCloud KVM backed instance.""" - - platform_name = "nocloud-kvm" - _ssh_client = None - - def __init__(self, platform, name, image_path, properties, config, - features, user_data, meta_data): - """Set up instance. - - @param platform: platform object - @param name: image path - @param image_path: path to disk image to boot. - @param properties: dictionary of properties - @param config: dictionary of configuration values - @param features: dictionary of supported feature flags - """ - self.user_data = user_data - self.meta_data = meta_data - self.ssh_key_file = os.path.join(platform.config['data_dir'], - platform.config['private_key']) - self.ssh_port = None - self.pid = None - self.pid_file = None - self.console_file = None - self.disk = image_path - - super(NoCloudKVMInstance, self).__init__( - platform, name, properties, config, features) - - def destroy(self): - """Clean up instance.""" - if self.pid: - try: - c_util.subp(['kill', '-9', self.pid]) - except util.ProcessExectuionError: - pass - - if self.pid_file: - os.remove(self.pid_file) - - self.pid = None - if self._ssh_client: - self._ssh_client.close() - self._ssh_client = None - - super(NoCloudKVMInstance, self).destroy() - - def _execute(self, command, stdin=None, env=None): - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - return self.ssh(['sudo'] + env_args + list(command), stdin=stdin) - - def generate_seed(self, tmpdir): - """Generate nocloud seed from user-data""" - seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) - user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) - - with open(user_data_file, "w") as ud_file: - ud_file.write(self.user_data) - - c_util.subp(['cloud-localds', seed_file, user_data_file]) - - return seed_file - - def get_free_port(self): - """Get a free port assigned by the kernel.""" - s = socket.socket() - s.bind(('', 0)) - num = s.getsockname()[1] - s.close() - return num - - def ssh(self, command, stdin=None): - """Run a command via SSH.""" - client = self._ssh_connect() - - cmd = util.shell_pack(command) - try: - fp_in, fp_out, fp_err = client.exec_command(cmd) - channel = fp_in.channel - if stdin is not None: - fp_in.write(stdin) - fp_in.close() - - channel.shutdown_write() - rc = channel.recv_exit_status() - return (fp_out.read(), fp_err.read(), rc) - except paramiko.SSHException as e: - raise util.InTargetExecuteError( - b'', b'', -1, command, self.name, reason=e) - - def _ssh_connect(self, hostname='localhost', username='ubuntu', - banner_timeout=120, retry_attempts=30): - """Connect via SSH.""" - if self._ssh_client: - return self._ssh_client - - private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - while retry_attempts: - try: - client.connect(hostname=hostname, username=username, - port=self.ssh_port, pkey=private_key, - banner_timeout=banner_timeout) - self._ssh_client = client - return client - except (paramiko.SSHException, TypeError): - time.sleep(1) - retry_attempts = retry_attempts - 1 - - error_desc = 'Failed command to: %s@%s:%s' % (username, hostname, - self.ssh_port) - raise util.InTargetExecuteError('', '', -1, 'ssh connect', - self.name, error_desc) - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - tmpdir = self.platform.config['data_dir'] - seed = self.generate_seed(tmpdir) - self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) - self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name) - self.ssh_port = self.get_free_port() - - cmd = ['./tools/xkvm', - '--disk', '%s,cache=unsafe' % self.disk, - '--disk', '%s,cache=unsafe' % seed, - '--netdev', ','.join(['user', - 'hostfwd=tcp::%s-:22' % self.ssh_port, - 'dnssearch=%s' % CI_DOMAIN]), - '--', '-pidfile', self.pid_file, '-vnc', 'none', - '-m', '2G', '-smp', '2', '-nographic', - '-serial', 'file:' + self.console_file] - subprocess.Popen(cmd, - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - while not os.path.exists(self.pid_file): - time.sleep(1) - - with open(self.pid_file, 'r') as pid_f: - self.pid = pid_f.readlines()[0].strip() - - if wait: - self._wait_for_system(wait_for_cloud_init) - - def console_log(self): - if not self.console_file: - return b'' - with open(self.console_file, "rb") as fp: - return fp.read() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index 3490fe87..92ed1627 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -2,8 +2,8 @@ """Main init.""" -from tests.cloud_tests.platforms import lxd -from tests.cloud_tests.platforms import nocloudkvm +from .lxd import platform as lxd +from .nocloudkvm import platform as nocloudkvm PLATFORMS = { 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, @@ -11,6 +11,16 @@ PLATFORMS = { } +def get_image(platform, config): + """Get image from platform object using os_name.""" + return platform.get_image(config) + + +def get_instance(snapshot, *args, **kwargs): + """Get instance from snapshot.""" + return snapshot.launch(*args, **kwargs) + + def get_platform(platform_name, config): """Get the platform object for 'platform_name' and init.""" platform_cls = PLATFORMS.get(platform_name) @@ -18,4 +28,10 @@ def get_platform(platform_name, config): raise ValueError('invalid platform name: {}'.format(platform_name)) return platform_cls(config) + +def get_snapshot(image): + """Get snapshot from image.""" + return image.snapshot() + + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/base.py b/tests/cloud_tests/platforms/base.py deleted file mode 100644 index 28975368..00000000 --- a/tests/cloud_tests/platforms/base.py +++ /dev/null @@ -1,27 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base platform class.""" - - -class Platform(object): - """Base class for platforms.""" - - platform_name = None - - def __init__(self, config): - """Set up platform.""" - self.config = config - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - raise NotImplementedError - - def destroy(self): - """Clean up platform data.""" - pass - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py new file mode 100644 index 00000000..d503108a --- /dev/null +++ b/tests/cloud_tests/platforms/images.py @@ -0,0 +1,56 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base class for images.""" + +from ..util import TargetBase + + +class Image(TargetBase): + """Base class for images.""" + + platform_name = None + + def __init__(self, platform, config): + """Set up image. + + @param platform: platform object + @param config: image configuration + """ + self.platform = platform + self.config = config + + def __str__(self): + """A brief description of the image.""" + return '-'.join((self.properties['os'], self.properties['release'])) + + @property + def properties(self): + """{} containing: 'arch', 'os', 'version', 'release'.""" + raise NotImplementedError + + @property + def features(self): + """Feature flags supported by this image. + + @return_value: list of feature names + """ + return [k for k, v in self.config.get('features', {}).items() if v] + + @property + def setup_overrides(self): + """Setup options that need to be overridden for the image. + + @return_value: dictionary to update args with + """ + # NOTE: more sophisticated options may be requied at some point + return self.config.get('setup_overrides', {}) + + def snapshot(self): + """Create snapshot of image, block until done.""" + raise NotImplementedError + + def destroy(self): + """Clean up data associated with image.""" + pass + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py new file mode 100644 index 00000000..8c59d62c --- /dev/null +++ b/tests/cloud_tests/platforms/instances.py @@ -0,0 +1,77 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base instance.""" + +from ..util import TargetBase + + +class Instance(TargetBase): + """Base instance object.""" + + platform_name = None + + def __init__(self, platform, name, properties, config, features): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.platform = platform + self.name = name + self.properties = properties + self.config = config + self.features = features + self._tmp_count = 0 + + def console_log(self): + """Instance console. + + @return_value: bytes of this instance’s console + """ + raise NotImplementedError + + def reboot(self, wait=True): + """Reboot instance.""" + raise NotImplementedError + + def shutdown(self, wait=True): + """Shutdown instance.""" + raise NotImplementedError + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + raise NotImplementedError + + def destroy(self): + """Clean up instance.""" + pass + + def _wait_for_system(self, wait_for_cloud_init): + """Wait until system has fully booted and cloud-init has finished. + + @param wait_time: maximum time to wait + @return_value: None, may raise OSError if wait_time exceeded + """ + def clean_test(test): + """Clean formatting for system ready test testcase.""" + return ' '.join(l for l in test.strip().splitlines() + if not l.lstrip().startswith('#')) + + time = self.config['boot_timeout'] + tests = [self.config['system_ready_script']] + if wait_for_cloud_init: + tests.append(self.config['cloud_init_ready_script']) + + formatted_tests = ' && '.join(clean_test(t) for t in tests) + cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' + 'exit 0; sleep 1; done; exit 1').format(time=time, + test=formatted_tests) + + if self.execute(cmd, rcs=(0, 1))[-1] != 0: + raise OSError('timeout: after {}s system not started'.format(time)) + + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd.py b/tests/cloud_tests/platforms/lxd.py deleted file mode 100644 index ead0955b..00000000 --- a/tests/cloud_tests/platforms/lxd.py +++ /dev/null @@ -1,108 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD platform.""" - -from pylxd import (Client, exceptions) - -from tests.cloud_tests.images import lxd as lxd_image -from tests.cloud_tests.instances import lxd as lxd_instance -from tests.cloud_tests.platforms import base -from tests.cloud_tests import util - -DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443" - - -class LXDPlatform(base.Platform): - """LXD test platform.""" - - platform_name = 'lxd' - - def __init__(self, config): - """Set up platform.""" - super(LXDPlatform, self).__init__(config) - # TODO: allow configuration of remote lxd host via env variables - # set up lxd connection - self.client = Client() - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - pylxd_image = self.client.images.create_from_simplestreams( - img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), - img_conf['alias']) - image = lxd_image.LXDImage(self, img_conf, pylxd_image) - if img_conf.get('override_templates', False): - image.update_templates(self.config.get('template_overrides', {}), - self.config.get('template_files', {})) - return image - - def launch_container(self, properties, config, features, - image=None, container=None, ephemeral=False, - container_config=None, block=True, image_desc=None, - use_desc=None): - """Launch a container. - - @param properties: image properties - @param config: image configuration - @param features: image features - @param image: image fingerprint to launch from - @param container: container to copy - @param ephemeral: delete image after first shutdown - @param container_config: config options for instance as dict - @param block: wait until container created - @param image_desc: description of image being launched - @param use_desc: description of container's use - @return_value: cloud_tests.instances instance - """ - if not (image or container): - raise ValueError("either image or container must be specified") - container = self.client.containers.create({ - 'name': util.gen_instance_name(image_desc=image_desc, - use_desc=use_desc, - used_list=self.list_containers()), - 'ephemeral': bool(ephemeral), - 'config': (container_config - if isinstance(container_config, dict) else {}), - 'source': ({'type': 'image', 'fingerprint': image} if image else - {'type': 'copy', 'source': container}) - }, wait=block) - return lxd_instance.LXDInstance(self, container.name, properties, - config, features, container) - - def container_exists(self, container_name): - """Check if container with name 'container_name' exists. - - @return_value: True if exists else False - """ - res = True - try: - self.client.containers.get(container_name) - except exceptions.LXDAPIException as e: - res = False - if e.response.status_code != 404: - raise - return res - - def list_containers(self): - """List names of all containers. - - @return_value: list of names - """ - return [container.name for container in self.client.containers.all()] - - def query_image_by_alias(self, alias): - """Get image by alias in local image store. - - @param alias: alias of image - @return_value: pylxd image (not cloud_tests.images instance) - """ - return self.client.images.get_by_alias(alias) - - def destroy(self): - """Clean up platform data.""" - super(LXDPlatform, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py new file mode 100644 index 00000000..b5de1f52 --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/image.py @@ -0,0 +1,193 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""LXD Image Base Class.""" + +import os +import shutil +import tempfile + +from ..images import Image +from .snapshot import LXDSnapshot +from cloudinit import util as c_util +from tests.cloud_tests import util + + +class LXDImage(Image): + """LXD backed image.""" + + platform_name = "lxd" + + def __init__(self, platform, config, pylxd_image): + """Set up image. + + @param platform: platform object + @param config: image configuration + """ + self.modified = False + self._img_instance = None + self._pylxd_image = None + self.pylxd_image = pylxd_image + super(LXDImage, self).__init__(platform, config) + + @property + def pylxd_image(self): + """Property function.""" + if self._pylxd_image: + self._pylxd_image.sync() + return self._pylxd_image + + @pylxd_image.setter + def pylxd_image(self, pylxd_image): + if self._img_instance: + self._instance.destroy() + self._img_instance = None + if (self._pylxd_image and + (self._pylxd_image is not pylxd_image) and + (not self.config.get('cache_base_image') or self.modified)): + self._pylxd_image.delete(wait=True) + self.modified = False + self._pylxd_image = pylxd_image + + @property + def _instance(self): + """Internal use only, returns a instance + + This starts an lxc instance from the image, so it is "dirty". + Better would be some way to modify this "at rest". + lxc-pstart would be an option.""" + if not self._img_instance: + self._img_instance = self.platform.launch_container( + self.properties, self.config, self.features, + use_desc='image-modification', image_desc=str(self), + image=self.pylxd_image.fingerprint) + self._img_instance.start() + return self._img_instance + + @property + def properties(self): + """{} containing: 'arch', 'os', 'version', 'release'.""" + properties = self.pylxd_image.properties + return { + 'arch': properties.get('architecture'), + 'os': properties.get('os'), + 'version': properties.get('version'), + 'release': properties.get('release'), + } + + def export_image(self, output_dir): + """Export image from lxd image store to (split) tarball on disk. + + @param output_dir: dir to store tarballs in + @return_value: tuple of path to metadata tarball and rootfs tarball + """ + # pylxd's image export feature doesn't do split exports, so use cmdline + c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, + output_dir], capture=True) + tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] + metadata = os.path.join( + output_dir, next(p for p in tarballs if p.startswith('meta-'))) + rootfs = os.path.join( + output_dir, next(p for p in tarballs if not p.startswith('meta-'))) + return (metadata, rootfs) + + def import_image(self, metadata, rootfs): + """Import image to lxd image store from (split) tarball on disk. + + Note, this will replace and delete the current pylxd_image + + @param metadata: metadata tarball + @param rootfs: rootfs tarball + @return_value: imported image fingerprint + """ + alias = util.gen_instance_name( + image_desc=str(self), use_desc='update-metadata') + c_util.subp(['lxc', 'image', 'import', metadata, rootfs, + '--alias', alias], capture=True) + self.pylxd_image = self.platform.query_image_by_alias(alias) + return self.pylxd_image.fingerprint + + def update_templates(self, template_config, template_data): + """Update the image's template configuration. + + Note, this will replace and delete the current pylxd_image + + @param template_config: config overrides for template metadata + @param template_data: template data to place into templates/ + """ + # set up tmp files + export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') + metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') + template_dir = os.path.join(extract_dir, 'templates') + + try: + # extract old data + (metadata, rootfs) = self.export_image(export_dir) + shutil.unpack_archive(metadata, extract_dir) + + # update metadata + metadata = c_util.read_conf(metadata_yaml) + templates = metadata.get('templates', {}) + templates.update(template_config) + metadata['templates'] = templates + util.yaml_dump(metadata, metadata_yaml) + + # write out template files + for name, content in template_data.items(): + path = os.path.join(template_dir, name) + c_util.write_file(path, content) + + # store new data, mark new image as modified + util.flat_tar(new_metadata, extract_dir) + self.import_image(new_metadata, rootfs) + self.modified = True + + finally: + # remove tmpfiles + shutil.rmtree(export_dir) + shutil.rmtree(extract_dir) + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done.""" + # get empty user data to pass in to instance + # if overrides for user data provided, use them + empty_userdata = util.update_user_data( + {}, self.config.get('user_data_overrides', {})) + conf = {'user.user-data': empty_userdata} + # clone current instance + instance = self.platform.launch_container( + self.properties, self.config, self.features, + container=self._instance.name, image_desc=str(self), + use_desc='snapshot', container_config=conf) + # wait for cloud-init before boot_clean_script is run to ensure + # /var/lib/cloud is removed cleanly + instance.start(wait=True, wait_for_cloud_init=True) + if self.config.get('boot_clean_script'): + instance.run_script(self.config.get('boot_clean_script')) + # freeze current instance and return snapshot + instance.freeze() + return LXDSnapshot(self.platform, self.properties, self.config, + self.features, instance) + + def destroy(self): + """Clean up data associated with image.""" + self.pylxd_image = None + super(LXDImage, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py new file mode 100644 index 00000000..0d697c05 --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -0,0 +1,157 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD instance.""" + +import os +import shutil +from tempfile import mkdtemp + +from ..instances import Instance + + +class LXDInstance(Instance): + """LXD container backed instance.""" + + platform_name = "lxd" + + def __init__(self, platform, name, properties, config, features, + pylxd_container): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self._pylxd_container = pylxd_container + super(LXDInstance, self).__init__( + platform, name, properties, config, features) + self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) + self._setup_console_log() + + @property + def pylxd_container(self): + """Property function.""" + self._pylxd_container.sync() + return self._pylxd_container + + def _setup_console_log(self): + logf = os.path.join(self.tmpd, "console.log") + + # doing this ensures we can read it. Otherwise it ends up root:root. + with open(logf, "w") as fp: + fp.write("# %s\n" % self.name) + + cfg = "lxc.console.logfile=%s" % logf + orig = self._pylxd_container.config.get('raw.lxc', "") + if orig: + orig += "\n" + self._pylxd_container.config['raw.lxc'] = orig + cfg + self._pylxd_container.save() + self._console_log_file = logf + + def _execute(self, command, stdin=None, env=None): + if env is None: + env = {} + + if stdin is not None: + # pylxd does not support input to execute. + # https://github.com/lxc/pylxd/issues/244 + # + # The solution here is write a tmp file in the container + # and then execute a shell that sets it standard in to + # be from that file, removes it, and calls the comand. + tmpf = self.tmpfile() + self.write_data(tmpf, stdin) + ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"' + command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] + + list(command)) + + # ensure instance is running and execute the command + self.start() + # execute returns a ContainerExecuteResult, named tuple + # (exit_code, stdout, stderr) + res = self.pylxd_container.execute(command, environment=env) + + # get out, exit and err from pylxd return + if not hasattr(res, 'exit_code'): + # pylxd 2.1.3 and earlier only return out and err, no exit + raise RuntimeError( + "No 'exit_code' in pylxd.container.execute return.\n" + "pylxd > 2.2 is required.") + + return res.stdout, res.stderr, res.exit_code + + def read_data(self, remote_path, decode=False): + """Read data from instance filesystem. + + @param remote_path: path in instance + @param decode: decode data before returning. + @return_value: content of remote_path as bytes if 'decode' is False, + and as string if 'decode' is True. + """ + data = self.pylxd_container.files.get(remote_path) + return data.decode() if decode else data + + def write_data(self, remote_path, data): + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write in bytes + """ + self.pylxd_container.files.put(remote_path, data) + + def console_log(self): + """Console log. + + @return_value: bytes of this instance’s console + """ + if not os.path.exists(self._console_log_file): + raise NotImplementedError( + "Console log '%s' does not exist. If this is a remote " + "lxc, then this is really NotImplementedError. If it is " + "A local lxc, then this is a RuntimeError." + "https://github.com/lxc/lxd/issues/1129") + with open(self._console_log_file, "rb") as fp: + return fp.read() + + def reboot(self, wait=True): + """Reboot instance.""" + self.shutdown(wait=wait) + self.start(wait=wait) + + def shutdown(self, wait=True): + """Shutdown instance.""" + if self.pylxd_container.status != 'Stopped': + self.pylxd_container.stop(wait=wait) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + if self.pylxd_container.status != 'Running': + self.pylxd_container.start(wait=wait) + if wait: + self._wait_for_system(wait_for_cloud_init) + + def freeze(self): + """Freeze instance.""" + if self.pylxd_container.status != 'Frozen': + self.pylxd_container.freeze(wait=True) + + def unfreeze(self): + """Unfreeze instance.""" + if self.pylxd_container.status == 'Frozen': + self.pylxd_container.unfreeze(wait=True) + + def destroy(self): + """Clean up instance.""" + self.unfreeze() + self.shutdown() + self.pylxd_container.delete(wait=True) + if self.platform.container_exists(self.name): + raise OSError('container {} was not properly removed' + .format(self.name)) + shutil.rmtree(self.tmpd) + super(LXDInstance, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py new file mode 100644 index 00000000..6a016929 --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/platform.py @@ -0,0 +1,108 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD platform.""" + +from pylxd import (Client, exceptions) + +from ..platforms import Platform +from .image import LXDImage +from .instance import LXDInstance +from tests.cloud_tests import util + +DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443" + + +class LXDPlatform(Platform): + """LXD test platform.""" + + platform_name = 'lxd' + + def __init__(self, config): + """Set up platform.""" + super(LXDPlatform, self).__init__(config) + # TODO: allow configuration of remote lxd host via env variables + # set up lxd connection + self.client = Client() + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + pylxd_image = self.client.images.create_from_simplestreams( + img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), + img_conf['alias']) + image = LXDImage(self, img_conf, pylxd_image) + if img_conf.get('override_templates', False): + image.update_templates(self.config.get('template_overrides', {}), + self.config.get('template_files', {})) + return image + + def launch_container(self, properties, config, features, + image=None, container=None, ephemeral=False, + container_config=None, block=True, image_desc=None, + use_desc=None): + """Launch a container. + + @param properties: image properties + @param config: image configuration + @param features: image features + @param image: image fingerprint to launch from + @param container: container to copy + @param ephemeral: delete image after first shutdown + @param container_config: config options for instance as dict + @param block: wait until container created + @param image_desc: description of image being launched + @param use_desc: description of container's use + @return_value: cloud_tests.instances instance + """ + if not (image or container): + raise ValueError("either image or container must be specified") + container = self.client.containers.create({ + 'name': util.gen_instance_name(image_desc=image_desc, + use_desc=use_desc, + used_list=self.list_containers()), + 'ephemeral': bool(ephemeral), + 'config': (container_config + if isinstance(container_config, dict) else {}), + 'source': ({'type': 'image', 'fingerprint': image} if image else + {'type': 'copy', 'source': container}) + }, wait=block) + return LXDInstance(self, container.name, properties, config, features, + container) + + def container_exists(self, container_name): + """Check if container with name 'container_name' exists. + + @return_value: True if exists else False + """ + res = True + try: + self.client.containers.get(container_name) + except exceptions.LXDAPIException as e: + res = False + if e.response.status_code != 404: + raise + return res + + def list_containers(self): + """List names of all containers. + + @return_value: list of names + """ + return [container.name for container in self.client.containers.all()] + + def query_image_by_alias(self, alias): + """Get image by alias in local image store. + + @param alias: alias of image + @return_value: pylxd image (not cloud_tests.images instance) + """ + return self.client.images.get_by_alias(alias) + + def destroy(self): + """Clean up platform data.""" + super(LXDPlatform, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/snapshot.py b/tests/cloud_tests/platforms/lxd/snapshot.py new file mode 100644 index 00000000..b524644f --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/snapshot.py @@ -0,0 +1,53 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD snapshot.""" + +from ..snapshots import Snapshot + + +class LXDSnapshot(Snapshot): + """LXD image copy backed snapshot.""" + + platform_name = "lxd" + + def __init__(self, platform, properties, config, features, + pylxd_frozen_instance): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.pylxd_frozen_instance = pylxd_frozen_instance + super(LXDSnapshot, self).__init__( + platform, properties, config, features) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + inst_config = {'user.user-data': user_data} + if meta_data: + inst_config['user.meta-data'] = meta_data + instance = self.platform.launch_container( + self.properties, self.config, self.features, block=block, + image_desc=str(self), container=self.pylxd_frozen_instance.name, + use_desc=use_desc, container_config=inst_config) + if start: + instance.start() + return instance + + def destroy(self): + """Clean up snapshot data.""" + self.pylxd_frozen_instance.destroy() + super(LXDSnapshot, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm.py b/tests/cloud_tests/platforms/nocloudkvm.py deleted file mode 100644 index 76cd83ad..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm.py +++ /dev/null @@ -1,91 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM platform.""" -import glob -import os - -from simplestreams import filters -from simplestreams import mirrors -from simplestreams import objectstores -from simplestreams import util as s_util - -from cloudinit import util as c_util -from tests.cloud_tests.images import nocloudkvm as nocloud_kvm_image -from tests.cloud_tests.instances import nocloudkvm as nocloud_kvm_instance -from tests.cloud_tests.platforms import base -from tests.cloud_tests import util - - -class NoCloudKVMPlatform(base.Platform): - """NoCloud KVM test platform.""" - - platform_name = 'nocloud-kvm' - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) - - filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), - 'release=%s' % img_conf['release'], - 'ftype=disk1.img']) - mirror_config = {'filters': filter, - 'keep_items': False, - 'max_items': 1, - 'checksumming_reader': True, - 'item_download': True - } - - def policy(content, path): - return s_util.read_signed(content, keyring=img_conf['keyring']) - - smirror = mirrors.UrlMirrorReader(url, policy=policy) - tstore = objectstores.FileStore(img_conf['mirror_dir']) - tmirror = mirrors.ObjectFilterMirror(config=mirror_config, - objectstore=tstore) - tmirror.sync(smirror, path) - - search_d = os.path.join(img_conf['mirror_dir'], '**', - img_conf['release'], '**', '*.img') - - images = [] - for fname in glob.iglob(search_d, recursive=True): - images.append(fname) - - if len(images) < 1: - raise RuntimeError("No images found under '%s'" % search_d) - if len(images) > 1: - raise RuntimeError( - "Multiple images found in '%s': %s" % (search_d, - ' '.join(images))) - - image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) - return image - - def create_instance(self, properties, config, features, - src_img_path, image_desc=None, use_desc=None, - user_data=None, meta_data=None): - """Create an instance - - @param src_img_path: image path to launch from - @param properties: image properties - @param config: image configuration - @param features: image features - @param image_desc: description of image being launched - @param use_desc: description of container's use - @return_value: cloud_tests.instances instance - """ - name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) - img_path = os.path.join(self.config['data_dir'], name + '.qcow2') - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', src_img_path, img_path]) - - return nocloud_kvm_instance.NoCloudKVMInstance(self, name, img_path, - properties, config, - features, user_data, - meta_data) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py new file mode 100644 index 00000000..09ff2a3b --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -0,0 +1,89 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""NoCloud KVM Image Base Class.""" + +from cloudinit import util as c_util + +import os +import shutil +import tempfile + +from ..images import Image +from .snapshot import NoCloudKVMSnapshot + + +class NoCloudKVMImage(Image): + """NoCloud KVM backed image.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, config, orig_img_path): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param img_path: path to the image + """ + self.modified = False + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage') + self._orig_img_path = orig_img_path + self._img_path = os.path.join(self._workd, + os.path.basename(self._orig_img_path)) + + c_util.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', orig_img_path, self._img_path]) + + super(NoCloudKVMImage, self).__init__(platform, config) + + @property + def properties(self): + """Dictionary containing: 'arch', 'os', 'version', 'release'.""" + return { + 'arch': self.config['arch'], + 'os': self.config['family'], + 'release': self.config['release'], + 'version': self.config['version'], + } + + def _execute(self, command, stdin=None, env=None): + """Execute command in image, modifying image.""" + return self.mount_image_callback(command, stdin=stdin, env=env) + + def mount_image_callback(self, command, stdin=None, env=None): + """Run mount-image-callback.""" + + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts', + '--system-resolvconf', self._img_path, + '--', 'chroot', '_MOUNTPOINT_'] + try: + out, err = c_util.subp(mic_chroot + env_args + list(command), + data=stdin, decode=False) + return (out, err, 0) + except c_util.ProcessExecutionError as e: + return (e.stdout, e.stderr, e.exit_code) + + def snapshot(self): + """Create snapshot of image, block until done.""" + if not self._img_path: + raise RuntimeError() + + return NoCloudKVMSnapshot(self.platform, self.properties, self.config, + self.features, self._img_path) + + def destroy(self): + """Unset path to signal image is no longer used. + + The removal of the images and all other items is handled by the + framework. In some cases we want to keep the images, so let the + framework decide whether to keep or destroy everything. + """ + self._img_path = None + shutil.rmtree(self._workd) + + super(NoCloudKVMImage, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py new file mode 100644 index 00000000..a87d76a6 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -0,0 +1,179 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM instance.""" + +import os +import paramiko +import socket +import subprocess +import time + +from ..instances import Instance +from cloudinit import util as c_util +from tests.cloud_tests import util + +# This domain contains reverse lookups for hostnames that are used. +# The primary reason is so sudo will return quickly when it attempts +# to look up the hostname. i9n is just short for 'integration'. +# see also bug 1730744 for why we had to do this. +CI_DOMAIN = "i9n.cloud-init.io" + + +class NoCloudKVMInstance(Instance): + """NoCloud KVM backed instance.""" + + platform_name = "nocloud-kvm" + _ssh_client = None + + def __init__(self, platform, name, image_path, properties, config, + features, user_data, meta_data): + """Set up instance. + + @param platform: platform object + @param name: image path + @param image_path: path to disk image to boot. + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + """ + self.user_data = user_data + self.meta_data = meta_data + self.ssh_key_file = os.path.join(platform.config['data_dir'], + platform.config['private_key']) + self.ssh_port = None + self.pid = None + self.pid_file = None + self.console_file = None + self.disk = image_path + + super(NoCloudKVMInstance, self).__init__( + platform, name, properties, config, features) + + def destroy(self): + """Clean up instance.""" + if self.pid: + try: + c_util.subp(['kill', '-9', self.pid]) + except util.ProcessExectuionError: + pass + + if self.pid_file: + os.remove(self.pid_file) + + self.pid = None + if self._ssh_client: + self._ssh_client.close() + self._ssh_client = None + + super(NoCloudKVMInstance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self.ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def generate_seed(self, tmpdir): + """Generate nocloud seed from user-data""" + seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) + user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) + + with open(user_data_file, "w") as ud_file: + ud_file.write(self.user_data) + + c_util.subp(['cloud-localds', seed_file, user_data_file]) + + return seed_file + + def get_free_port(self): + """Get a free port assigned by the kernel.""" + s = socket.socket() + s.bind(('', 0)) + num = s.getsockname()[1] + s.close() + return num + + def ssh(self, command, stdin=None): + """Run a command via SSH.""" + client = self._ssh_connect() + + cmd = util.shell_pack(command) + try: + fp_in, fp_out, fp_err = client.exec_command(cmd) + channel = fp_in.channel + if stdin is not None: + fp_in.write(stdin) + fp_in.close() + + channel.shutdown_write() + rc = channel.recv_exit_status() + return (fp_out.read(), fp_err.read(), rc) + except paramiko.SSHException as e: + raise util.InTargetExecuteError( + b'', b'', -1, command, self.name, reason=e) + + def _ssh_connect(self, hostname='localhost', username='ubuntu', + banner_timeout=120, retry_attempts=30): + """Connect via SSH.""" + if self._ssh_client: + return self._ssh_client + + private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + while retry_attempts: + try: + client.connect(hostname=hostname, username=username, + port=self.ssh_port, pkey=private_key, + banner_timeout=banner_timeout) + self._ssh_client = client + return client + except (paramiko.SSHException, TypeError): + time.sleep(1) + retry_attempts = retry_attempts - 1 + + error_desc = 'Failed command to: %s@%s:%s' % (username, hostname, + self.ssh_port) + raise util.InTargetExecuteError('', '', -1, 'ssh connect', + self.name, error_desc) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + tmpdir = self.platform.config['data_dir'] + seed = self.generate_seed(tmpdir) + self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) + self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name) + self.ssh_port = self.get_free_port() + + cmd = ['./tools/xkvm', + '--disk', '%s,cache=unsafe' % self.disk, + '--disk', '%s,cache=unsafe' % seed, + '--netdev', ','.join(['user', + 'hostfwd=tcp::%s-:22' % self.ssh_port, + 'dnssearch=%s' % CI_DOMAIN]), + '--', '-pidfile', self.pid_file, '-vnc', 'none', + '-m', '2G', '-smp', '2', '-nographic', + '-serial', 'file:' + self.console_file] + subprocess.Popen(cmd, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + while not os.path.exists(self.pid_file): + time.sleep(1) + + with open(self.pid_file, 'r') as pid_f: + self.pid = pid_f.readlines()[0].strip() + + if wait: + self._wait_for_system(wait_for_cloud_init) + + def console_log(self): + if not self.console_file: + return b'' + with open(self.console_file, "rb") as fp: + return fp.read() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py new file mode 100644 index 00000000..85933463 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -0,0 +1,89 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM platform.""" +import glob +import os + +from simplestreams import filters +from simplestreams import mirrors +from simplestreams import objectstores +from simplestreams import util as s_util + +from ..platforms import Platform +from .image import NoCloudKVMImage +from .instance import NoCloudKVMInstance +from cloudinit import util as c_util +from tests.cloud_tests import util + + +class NoCloudKVMPlatform(Platform): + """NoCloud KVM test platform.""" + + platform_name = 'nocloud-kvm' + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) + + filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), + 'release=%s' % img_conf['release'], + 'ftype=disk1.img']) + mirror_config = {'filters': filter, + 'keep_items': False, + 'max_items': 1, + 'checksumming_reader': True, + 'item_download': True + } + + def policy(content, path): + return s_util.read_signed(content, keyring=img_conf['keyring']) + + smirror = mirrors.UrlMirrorReader(url, policy=policy) + tstore = objectstores.FileStore(img_conf['mirror_dir']) + tmirror = mirrors.ObjectFilterMirror(config=mirror_config, + objectstore=tstore) + tmirror.sync(smirror, path) + + search_d = os.path.join(img_conf['mirror_dir'], '**', + img_conf['release'], '**', '*.img') + + images = [] + for fname in glob.iglob(search_d, recursive=True): + images.append(fname) + + if len(images) < 1: + raise RuntimeError("No images found under '%s'" % search_d) + if len(images) > 1: + raise RuntimeError( + "Multiple images found in '%s': %s" % (search_d, + ' '.join(images))) + + image = NoCloudKVMImage(self, img_conf, images[0]) + return image + + def create_instance(self, properties, config, features, + src_img_path, image_desc=None, use_desc=None, + user_data=None, meta_data=None): + """Create an instance + + @param src_img_path: image path to launch from + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_desc: description of image being launched + @param use_desc: description of container's use + @return_value: cloud_tests.instances instance + """ + name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) + img_path = os.path.join(self.config['data_dir'], name + '.qcow2') + c_util.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', src_img_path, img_path]) + + return NoCloudKVMInstance(self, name, img_path, properties, config, + features, user_data, meta_data) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py new file mode 100644 index 00000000..0005e1f2 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py @@ -0,0 +1,79 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM snapshot.""" +import os +import shutil +import tempfile + +from ..snapshots import Snapshot + + +class NoCloudKVMSnapshot(Snapshot): + """NoCloud KVM image copy backed snapshot.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, properties, config, features, image_path): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + @param image_path: image file to snapshot. + """ + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot') + snapshot = os.path.join(self._workd, 'snapshot') + shutil.copyfile(image_path, snapshot) + self._image_path = snapshot + + super(NoCloudKVMSnapshot, self).__init__( + platform, properties, config, features) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + key_file = os.path.join(self.platform.config['data_dir'], + self.platform.config['public_key']) + user_data = self.inject_ssh_key(user_data, key_file) + + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self._image_path, image_desc=str(self), use_desc=use_desc, + user_data=user_data, meta_data=meta_data) + + if start: + instance.start() + + return instance + + def inject_ssh_key(self, user_data, key_file): + """Inject the authorized key into the user_data.""" + with open(key_file) as f: + value = f.read() + + key = 'ssh_authorized_keys:' + value = ' - %s' % value.strip() + user_data = user_data.split('\n') + if key in user_data: + user_data.insert(user_data.index(key) + 1, '%s' % value) + else: + user_data.insert(-1, '%s' % key) + user_data.insert(-1, '%s' % value) + + return '\n'.join(user_data) + + def destroy(self): + """Clean up snapshot data.""" + shutil.rmtree(self._workd) + super(NoCloudKVMSnapshot, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py new file mode 100644 index 00000000..28975368 --- /dev/null +++ b/tests/cloud_tests/platforms/platforms.py @@ -0,0 +1,27 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base platform class.""" + + +class Platform(object): + """Base class for platforms.""" + + platform_name = None + + def __init__(self, config): + """Set up platform.""" + self.config = config + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + raise NotImplementedError + + def destroy(self): + """Clean up platform data.""" + pass + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py new file mode 100644 index 00000000..94328982 --- /dev/null +++ b/tests/cloud_tests/platforms/snapshots.py @@ -0,0 +1,45 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base snapshot.""" + + +class Snapshot(object): + """Base class for snapshots.""" + + platform_name = None + + def __init__(self, platform, properties, config, features): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.platform = platform + self.properties = properties + self.config = config + self.features = features + + def __str__(self): + """A brief description of the snapshot.""" + return '-'.join((self.properties['os'], self.properties['release'])) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + raise NotImplementedError + + def destroy(self): + """Clean up snapshot data.""" + pass + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/__init__.py b/tests/cloud_tests/snapshots/__init__.py deleted file mode 100644 index 93a54f5e..00000000 --- a/tests/cloud_tests/snapshots/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Main init.""" - - -def get_snapshot(image): - """Get snapshot from image.""" - return image.snapshot() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/base.py b/tests/cloud_tests/snapshots/base.py deleted file mode 100644 index 94328982..00000000 --- a/tests/cloud_tests/snapshots/base.py +++ /dev/null @@ -1,45 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base snapshot.""" - - -class Snapshot(object): - """Base class for snapshots.""" - - platform_name = None - - def __init__(self, platform, properties, config, features): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.platform = platform - self.properties = properties - self.config = config - self.features = features - - def __str__(self): - """A brief description of the snapshot.""" - return '-'.join((self.properties['os'], self.properties['release'])) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - raise NotImplementedError - - def destroy(self): - """Clean up snapshot data.""" - pass - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/lxd.py b/tests/cloud_tests/snapshots/lxd.py deleted file mode 100644 index 39c55c5e..00000000 --- a/tests/cloud_tests/snapshots/lxd.py +++ /dev/null @@ -1,53 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD snapshot.""" - -from tests.cloud_tests.snapshots import base - - -class LXDSnapshot(base.Snapshot): - """LXD image copy backed snapshot.""" - - platform_name = "lxd" - - def __init__(self, platform, properties, config, features, - pylxd_frozen_instance): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.pylxd_frozen_instance = pylxd_frozen_instance - super(LXDSnapshot, self).__init__( - platform, properties, config, features) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - inst_config = {'user.user-data': user_data} - if meta_data: - inst_config['user.meta-data'] = meta_data - instance = self.platform.launch_container( - self.properties, self.config, self.features, block=block, - image_desc=str(self), container=self.pylxd_frozen_instance.name, - use_desc=use_desc, container_config=inst_config) - if start: - instance.start() - return instance - - def destroy(self): - """Clean up snapshot data.""" - self.pylxd_frozen_instance.destroy() - super(LXDSnapshot, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/nocloudkvm.py b/tests/cloud_tests/snapshots/nocloudkvm.py deleted file mode 100644 index 21e908da..00000000 --- a/tests/cloud_tests/snapshots/nocloudkvm.py +++ /dev/null @@ -1,79 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM snapshot.""" -import os -import shutil -import tempfile - -from tests.cloud_tests.snapshots import base - - -class NoCloudKVMSnapshot(base.Snapshot): - """NoCloud KVM image copy backed snapshot.""" - - platform_name = "nocloud-kvm" - - def __init__(self, platform, properties, config, features, image_path): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - @param image_path: image file to snapshot. - """ - self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot') - snapshot = os.path.join(self._workd, 'snapshot') - shutil.copyfile(image_path, snapshot) - self._image_path = snapshot - - super(NoCloudKVMSnapshot, self).__init__( - platform, properties, config, features) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - key_file = os.path.join(self.platform.config['data_dir'], - self.platform.config['public_key']) - user_data = self.inject_ssh_key(user_data, key_file) - - instance = self.platform.create_instance( - self.properties, self.config, self.features, - self._image_path, image_desc=str(self), use_desc=use_desc, - user_data=user_data, meta_data=meta_data) - - if start: - instance.start() - - return instance - - def inject_ssh_key(self, user_data, key_file): - """Inject the authorized key into the user_data.""" - with open(key_file) as f: - value = f.read() - - key = 'ssh_authorized_keys:' - value = ' - %s' % value.strip() - user_data = user_data.split('\n') - if key in user_data: - user_data.insert(user_data.index(key) + 1, '%s' % value) - else: - user_data.insert(-1, '%s' % key) - user_data.insert(-1, '%s' % value) - - return '\n'.join(user_data) - - def destroy(self): - """Clean up snapshot data.""" - shutil.rmtree(self._workd) - super(NoCloudKVMSnapshot, self).destroy() - -# vi: ts=4 expandtab -- cgit v1.2.3 From a110e483e8644ab73e69853ea11b6c4c6cfa04b6 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 6 Dec 2017 16:30:22 -0600 Subject: pylint: Update pylint to 1.7.1, run on tests/ and tools and fix complaints. The motivation for this is that a.) 1.7.1 runs with python 3.6 (bionic) b.) we want to run pylint on tests/ and tools for the same reasons that we want to run it on cloudinit/ The changes are described below. - Update tox.ini to invoke pylint v1.7.1. - Modify .pylintrc generated-members ignore mocked object members (m_.*) - Replace "dangerous" params defaulting to {} - Fix up cloud_tests use of platforms - Cast some instance objects to with dict() - Handle python2.7 vs 3+ ConfigParser use of readfp (deprecated) - Update use of assertEqual(, value) to assert(value) - replace depricated assertRegexp -> assertRegex - Remove useless test-class calls to super class - Assign class property accessors a result and use it - Fix missing class member in CepkoResultTests - Fix Cheetah test import --- .pylintrc | 2 +- cloudinit/cmd/tests/test_clean.py | 2 +- cloudinit/cmd/tests/test_status.py | 2 +- cloudinit/tests/helpers.py | 35 +++++ tests/cloud_tests/__init__.py | 6 + tests/cloud_tests/bddeb.py | 9 +- tests/cloud_tests/collect.py | 6 +- tests/cloud_tests/config.py | 4 +- tests/cloud_tests/testcases/base.py | 3 +- .../testcases/modules/set_hostname_fqdn.py | 2 +- tests/cloud_tests/util.py | 2 +- tests/unittests/test_cs_util.py | 1 + tests/unittests/test_datasource/test_azure.py | 31 ++-- .../unittests/test_datasource/test_digitalocean.py | 9 +- tests/unittests/test_datasource/test_ec2.py | 3 +- tests/unittests/test_distros/test_create_users.py | 7 +- tests/unittests/test_distros/test_netconfig.py | 3 - tests/unittests/test_handler/test_handler_lxd.py | 3 - .../test_handler/test_handler_power_state.py | 3 - .../test_handler/test_handler_yum_add_repo.py | 10 +- .../test_handler/test_handler_zypper_add_repo.py | 7 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_templating.py | 2 +- tests/unittests/test_util.py | 6 +- tests/unittests/test_vmware_config_file.py | 3 +- tools/hacking.py | 172 --------------------- tools/make-mime.py | 2 +- tools/mock-meta.py | 45 +++--- tox.ini | 5 +- 29 files changed, 121 insertions(+), 266 deletions(-) delete mode 100755 tools/hacking.py (limited to 'tests/cloud_tests') diff --git a/.pylintrc b/.pylintrc index b160ce7b..3ad36924 100644 --- a/.pylintrc +++ b/.pylintrc @@ -56,5 +56,5 @@ ignored-classes=optparse.Values,thread._local # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members=types,http.client,command_handlers +generated-members=types,http.client,command_handlers,m_.* diff --git a/cloudinit/cmd/tests/test_clean.py b/cloudinit/cmd/tests/test_clean.py index af438aab..1379740b 100644 --- a/cloudinit/cmd/tests/test_clean.py +++ b/cloudinit/cmd/tests/test_clean.py @@ -151,7 +151,7 @@ class TestClean(CiTestCase): 'sys.argv': {'new': ['clean', '--logs']}}, clean.main) - self.assertEqual(0, context_manager.exception.code) + self.assertRaisesCodeEqual(0, context_manager.exception.code) self.assertFalse( os.path.exists(self.log1), 'Unexpected log {0}'.format(self.log1)) diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py index 8ec9b5bc..6d4a11e8 100644 --- a/cloudinit/cmd/tests/test_status.py +++ b/cloudinit/cmd/tests/test_status.py @@ -347,7 +347,7 @@ class TestStatus(CiTestCase): '_is_cloudinit_disabled': (False, ''), 'Init': {'side_effect': self.init_class}}, status.main) - self.assertEqual(0, context_manager.exception.code) + self.assertRaisesCodeEqual(0, context_manager.exception.code) self.assertEqual('status: running\n', m_stdout.getvalue()) # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index feb884ab..0080c729 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -19,6 +19,11 @@ try: except ImportError: from contextlib2 import ExitStack +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser + from cloudinit import helpers as ch from cloudinit import util @@ -113,6 +118,16 @@ class TestCase(unittest2.TestCase): self.addCleanup(m.stop) setattr(self, attr, p) + # prefer python3 read_file over readfp but allow fallback + def parse_and_read(self, contents): + parser = ConfigParser() + if hasattr(parser, 'read_file'): + parser.read_file(contents) + elif hasattr(parser, 'readfp'): + # pylint: disable=W1505 + parser.readfp(contents) + return parser + class CiTestCase(TestCase): """This is the preferred test case base class unless user @@ -158,6 +173,18 @@ class CiTestCase(TestCase): dir = self.tmp_dir() return os.path.normpath(os.path.abspath(os.path.join(dir, path))) + def assertRaisesCodeEqual(self, expected, found): + """Handle centos6 having different context manager for assertRaises. + with assertRaises(Exception) as e: + raise Exception("BOO") + + centos6 will have e.exception as an integer. + anything nwere will have it as something with a '.code'""" + if isinstance(found, int): + self.assertEqual(expected, found) + else: + self.assertEqual(expected, found.code) + class ResourceUsingTestCase(CiTestCase): @@ -395,4 +422,12 @@ if not hasattr(mock.Mock, 'assert_not_called'): mock.Mock.assert_not_called = __mock_assert_not_called +# older unittest2.TestCase (centos6) do not have assertRaisesRegex +# And setting assertRaisesRegex to assertRaisesRegexp causes +# https://github.com/PyCQA/pylint/issues/1653 . So the workaround. +if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): + def _tricky(*args, **kwargs): + return unittest2.TestCase.assertRaisesRegexp + unittest2.TestCase.assertRaisesRegex = _tricky + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/__init__.py b/tests/cloud_tests/__init__.py index 98c1d6c7..dd436989 100644 --- a/tests/cloud_tests/__init__.py +++ b/tests/cloud_tests/__init__.py @@ -10,6 +10,12 @@ TESTCASES_DIR = os.path.join(BASE_DIR, 'testcases') TEST_CONF_DIR = os.path.join(BASE_DIR, 'testcases') TREE_BASE = os.sep.join(BASE_DIR.split(os.sep)[:-2]) +# This domain contains reverse lookups for hostnames that are used. +# The primary reason is so sudo will return quickly when it attempts +# to look up the hostname. i9n is just short for 'integration'. +# see also bug 1730744 for why we had to do this. +CI_DOMAIN = "i9n.cloud-init.io" + def _initialize_logging(): """Configure logging for cloud_tests.""" diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index c259dfea..a6d5069f 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -8,8 +8,7 @@ import tempfile from cloudinit import util as c_util from tests.cloud_tests import (config, LOG) -from tests.cloud_tests.platforms import (platforms, images, snapshots, - instances) +from tests.cloud_tests import platforms from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) pre_reqs = ['devscripts', 'equivs', 'git', 'tar'] @@ -85,18 +84,18 @@ def setup_build(args): # set up image LOG.info('acquiring image for os: %s', args.build_os) img_conf = config.load_os_config(platform.platform_name, args.build_os) - image_call = partial(images.get_image, platform, img_conf) + image_call = partial(platforms.get_image, platform, img_conf) with PlatformComponent(image_call) as image: # set up snapshot - snapshot_call = partial(snapshots.get_snapshot, image) + snapshot_call = partial(platforms.get_snapshot, image) with PlatformComponent(snapshot_call) as snapshot: # create instance with cloud-config to set it up LOG.info('creating instance to build deb in') empty_cloud_config = "#cloud-config\n{}" instance_call = partial( - instances.get_instance, snapshot, empty_cloud_config, + platforms.get_instance, snapshot, empty_cloud_config, use_desc='build cloud-init deb') with PlatformComponent(instance_call) as instance: diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index db5ee99f..4805cea1 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -64,9 +64,9 @@ def collect_test_data(args, snapshot, os_name, test_name): # skip the testcase with a warning req_features = test_config.get('required_features', []) if any(feature not in snapshot.features for feature in req_features): - LOG.warn('test config %s requires features not supported by image, ' - 'skipping.\nrequired features: %s\nsupported features: %s', - test_name, req_features, snapshot.features) + LOG.warning('test config %s requires features not supported by image, ' + 'skipping.\nrequired features: %s\nsupported features: %s', + test_name, req_features, snapshot.features) return ({}, 0) # if there are user data overrides required for this test case, apply them diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py index 52fc2bda..8bd569fd 100644 --- a/tests/cloud_tests/config.py +++ b/tests/cloud_tests/config.py @@ -92,7 +92,7 @@ def load_platform_config(platform_name, require_enabled=False): def load_os_config(platform_name, os_name, require_enabled=False, - feature_overrides={}): + feature_overrides=None): """Load configuration for os. @param platform_name: platform name to load os config for @@ -101,6 +101,8 @@ def load_os_config(platform_name, os_name, require_enabled=False, @param feature_overrides: feature flag overrides to merge with features @return_value: config dict """ + if feature_overrides is None: + feature_overrides = {} main_conf = c_util.read_conf(RELEASES_CONF) default = main_conf['default_release_config'] image = main_conf['releases'][os_name] diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 1706f59b..1c5b5405 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -12,7 +12,8 @@ from cloudinit import util as c_util class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" - data = None + # data gets populated in get_suite.setUpClass + data = {} conf = None _cloud_config = None diff --git a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py index eb6f0650..a405b30b 100644 --- a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py +++ b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -from tests.cloud_tests.instances.nocloudkvm import CI_DOMAIN +from tests.cloud_tests import CI_DOMAIN from tests.cloud_tests.testcases import base diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index c5cd6974..2aedcd0d 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -262,7 +262,7 @@ def shell_safe(cmd): out = subprocess.check_output( ["getopt", "--shell", "sh", "--options", "", "--", "--"] + list(cmd)) # out contains ' -- \n'. drop the ' -- ' and the '\n' - return out[4:-1].decode() + return out.decode()[4:-1] def shell_pack(cmd): diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index ee88520d..2a1095b9 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -35,6 +35,7 @@ class CepkoMock(Cepko): # touched the underlying Cepko class methods. class CepkoResultTests(test_helpers.TestCase): def setUp(self): + self.c = Cepko() raise test_helpers.SkipTest('This test is completely useless') def test_getitem(self): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 226c214a..5ab48897 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -36,9 +36,9 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): """ for key, dval in data.items(): if isinstance(dval, dict): - val = dval.get('text') - attrs = ' ' + ' '.join(["%s='%s'" % (k, v) for k, v in dval.items() - if k != 'text']) + val = dict(dval).get('text') + attrs = ' ' + ' '.join(["%s='%s'" % (k, v) for k, v + in dict(dval).items() if k != 'text']) else: val = dval attrs = "" @@ -897,9 +897,6 @@ class TestCanDevBeReformatted(CiTestCase): setattr(self, sattr, patcher.start()) self.addCleanup(patcher.stop) - def setUp(self): - super(TestCanDevBeReformatted, self).setUp() - def patchup(self, devs): bypath = {} for path, data in devs.items(): @@ -954,14 +951,14 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda3': {'num': 3}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertFalse(False, value) + self.assertFalse(value) self.assertIn("3 or more", msg.lower()) def test_no_partitions_is_false(self): """A disk with no partitions can not be formatted.""" self.patchup({'/dev/sda': {}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(False, value) + self.assertFalse(value) self.assertIn("not partitioned", msg.lower()) def test_two_partitions_not_ntfs_false(self): @@ -973,7 +970,7 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda2': {'num': 2, 'fs': 'ext4', 'files': []}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertFalse(False, value) + self.assertFalse(value) self.assertIn("not ntfs", msg.lower()) def test_two_partitions_ntfs_populated_false(self): @@ -986,7 +983,7 @@ class TestCanDevBeReformatted(CiTestCase): 'files': ['secret.txt']}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertFalse(False, value) + self.assertFalse(value) self.assertIn("files on it", msg.lower()) def test_two_partitions_ntfs_empty_is_true(self): @@ -998,7 +995,7 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda2': {'num': 2, 'fs': 'ntfs', 'files': []}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(True, value) + self.assertTrue(value) self.assertIn("safe for", msg.lower()) def test_one_partition_not_ntfs_false(self): @@ -1009,7 +1006,7 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1, 'fs': 'zfs'}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(False, value) + self.assertFalse(value) self.assertIn("not ntfs", msg.lower()) def test_one_partition_ntfs_populated_false(self): @@ -1021,7 +1018,7 @@ class TestCanDevBeReformatted(CiTestCase): 'files': ['file1.txt', 'file2.exe']}, }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(False, value) + self.assertFalse(value) self.assertIn("files on it", msg.lower()) def test_one_partition_ntfs_empty_is_true(self): @@ -1032,7 +1029,7 @@ class TestCanDevBeReformatted(CiTestCase): '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []} }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(True, value) + self.assertTrue(value) self.assertIn("safe for", msg.lower()) def test_one_partition_ntfs_empty_with_dataloss_file_is_true(self): @@ -1044,7 +1041,7 @@ class TestCanDevBeReformatted(CiTestCase): 'files': ['dataloss_warning_readme.txt']} }}}) value, msg = dsaz.can_dev_be_reformatted("/dev/sda") - self.assertEqual(True, value) + self.assertTrue(value) self.assertIn("safe for", msg.lower()) def test_one_partition_through_realpath_is_true(self): @@ -1059,7 +1056,7 @@ class TestCanDevBeReformatted(CiTestCase): 'realpath': '/dev/sdb1'} }}}) value, msg = dsaz.can_dev_be_reformatted(epath) - self.assertEqual(True, value) + self.assertTrue(value) self.assertIn("safe for", msg.lower()) def test_three_partition_through_realpath_is_false(self): @@ -1078,7 +1075,7 @@ class TestCanDevBeReformatted(CiTestCase): 'realpath': '/dev/sdb3'} }}}) value, msg = dsaz.can_dev_be_reformatted(epath) - self.assertEqual(False, value) + self.assertFalse(value) self.assertIn("3 or more", msg.lower()) diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index ec321733..3127014b 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -199,9 +199,8 @@ class TestDataSourceDigitalOcean(CiTestCase): class TestNetworkConvert(CiTestCase): - @mock.patch('cloudinit.net.get_interfaces_by_mac') - def _get_networking(self, m_get_by_mac): - m_get_by_mac.return_value = { + def _get_networking(self): + self.m_get_by_mac.return_value = { '04:01:57:d1:9e:01': 'ens1', '04:01:57:d1:9e:02': 'ens2', 'b8:ae:ed:75:5f:9a': 'enp0s25', @@ -211,6 +210,10 @@ class TestNetworkConvert(CiTestCase): self.assertIn('config', netcfg) return netcfg + def setUp(self): + super(TestNetworkConvert, self).setUp() + self.add_patch('cloudinit.net.get_interfaces_by_mac', 'm_get_by_mac') + def test_networking_defined(self): netcfg = self._get_networking() self.assertIsNotNone(netcfg) diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index ba042eac..f0dc8338 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -330,7 +330,8 @@ class TestEc2(test_helpers.HttprettyTestCase): ds.fallback_nic = 'eth9' with mock.patch(get_interface_mac_path) as m_get_interface_mac: m_get_interface_mac.return_value = mac1 - ds.network_config # Will re-crawl network metadata + nc = ds.network_config # Will re-crawl network metadata + self.assertIsNotNone(nc) self.assertIn('Re-crawl of metadata service', self.logs.getvalue()) expected = {'version': 1, 'config': [ {'mac_address': '06:17:04:d7:26:09', diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index aa13670a..5670904a 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -7,7 +7,11 @@ from cloudinit.tests.helpers import (TestCase, mock) class MyBaseDistro(distros.Distro): # MyBaseDistro is here to test base Distro class implementations - def __init__(self, name="basedistro", cfg={}, paths={}): + def __init__(self, name="basedistro", cfg=None, paths=None): + if not cfg: + cfg = {} + if not paths: + paths = {} super(MyBaseDistro, self).__init__(name, cfg, paths) def install_packages(self, pkglist): @@ -42,7 +46,6 @@ class MyBaseDistro(distros.Distro): @mock.patch("cloudinit.distros.util.subp") class TestCreateUser(TestCase): def setUp(self): - super(TestCase, self).setUp() self.dist = MyBaseDistro() def _useradd2call(self, args): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index c4bd11bc..8d0b2634 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -188,9 +188,6 @@ hn0: flags=8843 metric 0 mtu 1500 status: active """ - def setUp(self): - super(TestNetCfgDistro, self).setUp() - def _get_distro(self, dname, renderers=None): cls = distros.fetch(dname) cfg = settings.CFG_BUILTIN diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index e0d9ab6c..a2054980 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -25,9 +25,6 @@ class TestLxd(t_help.CiTestCase): } } - def setUp(self): - super(TestLxd, self).setUp() - def _get_cloud(self, distro): cls = distros.fetch(distro) paths = helpers.Paths({}) diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 85a0fe0a..3c726422 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -9,9 +9,6 @@ from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): - def setUp(self): - super(self.__class__, self).setUp() - def test_no_config(self): # completely empty config should mean do nothing (cmd, _timeout, _condition) = psc.load_power_state({}) diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index b7adbe50..b90a3af3 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -5,10 +5,6 @@ from cloudinit import util from cloudinit.tests import helpers -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser import logging import shutil from six import StringIO @@ -58,8 +54,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): self.patchUtils(self.tmp) cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") - parser = ConfigParser() - parser.readfp(StringIO(contents)) + parser = self.parse_and_read(StringIO(contents)) expected = { 'epel_testing': { 'name': 'Extra Packages for Enterprise Linux 5 - Testing', @@ -95,8 +90,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): self.patchUtils(self.tmp) cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) contents = util.load_file("/etc/yum.repos.d/puppetlabs_products.repo") - parser = ConfigParser() - parser.readfp(StringIO(contents)) + parser = self.parse_and_read(StringIO(contents)) expected = { 'puppetlabs_products': { 'name': 'Puppet Labs Products El 6 - $basearch', diff --git a/tests/unittests/test_handler/test_handler_zypper_add_repo.py b/tests/unittests/test_handler/test_handler_zypper_add_repo.py index 315c2a5e..72ab6c08 100644 --- a/tests/unittests/test_handler/test_handler_zypper_add_repo.py +++ b/tests/unittests/test_handler/test_handler_zypper_add_repo.py @@ -9,10 +9,6 @@ from cloudinit import util from cloudinit.tests import helpers from cloudinit.tests.helpers import mock -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser import logging from six import StringIO @@ -70,8 +66,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): root_d = self.tmp_dir() cc_zypper_add_repo._write_repos(cfg['repos'], root_d) contents = util.load_file("%s/testing-foo.repo" % root_d) - parser = ConfigParser() - parser.readfp(StringIO(contents)) + parser = self.parse_and_read(StringIO(contents)) expected = { 'testing-foo': { 'name': 'test-foo', diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 571420ed..e15ba6cf 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -126,7 +126,7 @@ class TestBaseReportingHandler(TestCase): def test_base_reporting_handler_is_abstract(self): regexp = r".*abstract.*publish_event.*" - self.assertRaisesRegexp(TypeError, regexp, handlers.ReportingHandler) + self.assertRaisesRegex(TypeError, regexp, handlers.ReportingHandler) class TestLogHandler(TestCase): diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index b911d929..53154d33 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -14,7 +14,7 @@ from cloudinit import templater try: import Cheetah HAS_CHEETAH = True - Cheetah # make pyflakes happy, as Cheetah is not used here + c = Cheetah # make pyflakes and pylint happy, as Cheetah is not used here except ImportError: HAS_CHEETAH = False diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 71f59529..787ca208 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -695,9 +695,9 @@ class TestSubp(helpers.CiTestCase): util.write_file(noshebang, 'true\n') os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) - self.assertRaisesRegexp(util.ProcessExecutionError, - 'Missing #! in script\?', - util.subp, (noshebang,)) + self.assertRaisesRegex(util.ProcessExecutionError, + 'Missing #! in script\?', + util.subp, (noshebang,)) def test_returns_none_if_no_capture(self): (out, err) = util.subp(self.stdin2out, data=b'', capture=False) diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 808d303a..0f8cda95 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -133,7 +133,8 @@ class TestVmwareConfigFile(CiTestCase): conf = Config(cf) with self.assertRaises(ValueError): - conf.reset_password() + pw = conf.reset_password + self.assertIsNone(pw) cf.clear() cf._insertKey("PASSWORD|RESET", "yes") diff --git a/tools/hacking.py b/tools/hacking.py deleted file mode 100755 index e6a05136..00000000 --- a/tools/hacking.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012, Cloudscaling -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""cloudinit HACKING file compliance testing (based off of nova hacking.py) - -built on top of pep8.py -""" - -import inspect -import logging -import re -import sys - -import pep8 - -# Don't need this for testing -logging.disable('LOG') - -# N1xx comments -# N2xx except -# N3xx imports -# N4xx docstrings -# N[5-9]XX (future use) - -DOCSTRING_TRIPLE = ['"""', "'''"] -VERBOSE_MISSING_IMPORT = False -_missingImport = set([]) - - -def import_normalize(line): - # convert "from x import y" to "import x.y" - # handle "from x import y as z" to "import x.y as z" - split_line = line.split() - if (line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or (len(split_line) == 6 and - split_line[4] == "as"))): - return "import %s.%s" % (split_line[1], split_line[3]) - else: - return line - - -def cloud_import_alphabetical(physical_line, line_number, lines): - """Check for imports in alphabetical order. - - HACKING guide recommendation for imports: - imports in human alphabetical order - N306 - """ - # handle import x - # use .lower since capitalization shouldn't dictate order - split_line = import_normalize(physical_line.strip()).lower().split() - split_previous = import_normalize(lines[line_number - 2]) - split_previous = split_previous.strip().lower().split() - # with or without "as y" - length = [2, 4] - if (len(split_line) in length and len(split_previous) in length and - split_line[0] == "import" and split_previous[0] == "import"): - if split_line[1] < split_previous[1]: - return (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) - - -def cloud_docstring_start_space(physical_line): - """Check for docstring not start with space. - - HACKING guide recommendation for docstring: - Docstring should not start with space - N401 - """ - pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start - if (pos != -1 and len(physical_line) > pos + 1): - if (physical_line[pos + 3] == ' '): - return (pos, - "N401: one line docstring should not start with a space") - - -def cloud_todo_format(physical_line): - """Check for 'TODO()'. - - HACKING guide recommendation for TODO: - Include your name with TODOs as in "#TODO(termie)" - N101 - """ - pos = physical_line.find('TODO') - pos1 = physical_line.find('TODO(') - pos2 = physical_line.find('#') # make sure it's a comment - if (pos != pos1 and pos2 >= 0 and pos2 < pos): - return pos, "N101: Use TODO(NAME)" - - -def cloud_docstring_one_line(physical_line): - """Check one line docstring end. - - HACKING guide recommendation for one line docstring: - A one line docstring looks like this and ends in a period. - N402 - """ - pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start - end = max([physical_line[-4:-1] == i for i in DOCSTRING_TRIPLE]) # end - if (pos != -1 and end and len(physical_line) > pos + 4): - if (physical_line[-5] != '.'): - return pos, "N402: one line docstring needs a period" - - -def cloud_docstring_multiline_end(physical_line): - """Check multi line docstring end. - - HACKING guide recommendation for docstring: - Docstring should end on a new line - N403 - """ - pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start - if (pos != -1 and len(physical_line) == pos): - print(physical_line) - if (physical_line[pos + 3] == ' '): - return (pos, "N403: multi line docstring end on new line") - - -current_file = "" - - -def readlines(filename): - """Record the current file being tested.""" - pep8.current_file = filename - return open(filename).readlines() - - -def add_cloud(): - """Monkey patch pep8 for cloud-init guidelines. - - Look for functions that start with cloud_ - and add them to pep8 module. - - Assumes you know how to write pep8.py checks - """ - for name, function in globals().items(): - if not inspect.isfunction(function): - continue - if name.startswith("cloud_"): - exec("pep8.%s = %s" % (name, name)) - - -if __name__ == "__main__": - # NOVA based 'hacking.py' error codes start with an N - pep8.ERRORCODE_REGEX = re.compile(r'[EWN]\d{3}') - add_cloud() - pep8.current_file = current_file - pep8.readlines = readlines - try: - pep8._main() - finally: - if len(_missingImport) > 0: - sys.stderr.write( - "%i imports missing in this test environment\n" % - len(_missingImport)) - -# vi: ts=4 expandtab diff --git a/tools/make-mime.py b/tools/make-mime.py index f6a72044..d321479b 100755 --- a/tools/make-mime.py +++ b/tools/make-mime.py @@ -23,7 +23,7 @@ def file_content_type(text): filename, content_type = text.split(":", 1) return (open(filename, 'r'), filename, content_type.strip()) except ValueError: - raise argparse.ArgumentError("Invalid value for %r" % (text)) + raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) def main(): diff --git a/tools/mock-meta.py b/tools/mock-meta.py index a5d14ab7..724f7fc4 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -17,6 +17,7 @@ Then: ec2metadata --instance-id """ +import argparse import functools import json import logging @@ -27,8 +28,6 @@ import string import sys import yaml -from optparse import OptionParser - try: from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler import httplib as hclient @@ -415,29 +414,27 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'): def extract_opts(): - parser = OptionParser() - parser.add_option("-p", "--port", dest="port", action="store", type=int, - default=80, metavar="PORT", - help=("port from which to serve traffic" - " (default: %default)")) - parser.add_option("-a", "--addr", dest="address", action="store", type=str, - default='::', metavar="ADDRESS", - help=("address from which to serve traffic" - " (default: %default)")) - parser.add_option("-f", '--user-data-file', dest='user_data_file', - action='store', metavar='FILE', - help=("user data filename to serve back to" - "incoming requests")) - (options, args) = parser.parse_args() - out = dict() - out['extra'] = args - out['port'] = options.port - out['user_data_file'] = None - out['address'] = options.address - if options.user_data_file: - if not os.path.isfile(options.user_data_file): + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--port", dest="port", action="store", type=int, + default=80, metavar="PORT", + help=("port from which to serve traffic" + " (default: %default)")) + parser.add_argument("-a", "--addr", dest="address", action="store", + type=str, default='::', metavar="ADDRESS", + help=("address from which to serve traffic" + " (default: %default)")) + parser.add_argument("-f", '--user-data-file', dest='user_data_file', + action='store', metavar='FILE', + help=("user data filename to serve back to" + "incoming requests")) + parser.add_argument('extra', nargs='*') + args = parser.parse_args() + out = {'port': args.port, 'address': args.address, 'extra': args.extra, + 'user_data_file': None} + if args.user_data_file: + if not os.path.isfile(args.user_data_file): parser.error("Option -f specified a non-existent file") - with open(options.user_data_file, 'rb') as fh: + with open(args.user_data_file, 'rb') as fh: out['user_data_file'] = fh.read() return out diff --git a/tox.ini b/tox.ini index 92232201..d7316cc2 100644 --- a/tox.ini +++ b/tox.ini @@ -21,12 +21,13 @@ setenv = LC_ALL = en_US.utf-8 [testenv:pylint] +basepython = python3 deps = # requirements pylint==1.7.1 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt -commands = {envpython} -m pylint {posargs:cloudinit} +commands = {envpython} -m pylint {posargs:cloudinit tests tools} [testenv:py3] basepython = python3 @@ -119,7 +120,7 @@ commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} deps = pyflakes [testenv:tip-pylint] -commands = {envpython} -m pylint {posargs:cloudinit} +commands = {envpython} -m pylint {posargs:cloudinit tests tools} deps = # requirements pylint -- cgit v1.2.3 From 05b2308aa7e30337c2a455b5d2c67871b233e25c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Dec 2017 16:33:56 -0500 Subject: citest: In NoCloudKVM provide keys via metadata not userdata. The NoCloudKVM platform was inserting ssh keys via user-data rather than through meta-data like it is done on other platforms. This way we are not forced to change the user-data provided. Also, provide meta-data including a uuid as the instance-id. --- tests/cloud_tests/platforms/nocloudkvm/instance.py | 39 +++++++++++++++++++--- tests/cloud_tests/platforms/nocloudkvm/snapshot.py | 20 ----------- 2 files changed, 35 insertions(+), 24 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index a87d76a6..9bb24256 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -2,13 +2,16 @@ """Base NoCloud KVM instance.""" +import copy import os import paramiko import socket import subprocess import time +import uuid from ..instances import Instance +from cloudinit.atomic_helper import write_json from cloudinit import util as c_util from tests.cloud_tests import util @@ -37,14 +40,38 @@ class NoCloudKVMInstance(Instance): @param features: dictionary of supported feature flags """ self.user_data = user_data - self.meta_data = meta_data - self.ssh_key_file = os.path.join(platform.config['data_dir'], - platform.config['private_key']) + if meta_data: + meta_data = copy.deepcopy(meta_data) + else: + meta_data = {} + + if 'instance-id' in meta_data: + iid = meta_data['instance-id'] + else: + iid = str(uuid.uuid1()) + meta_data['instance-id'] = iid + + self.instance_id = iid + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + + self.ssh_pubkey = None + if self.ssh_pubkey_file: + with open(self.ssh_pubkey_file, "r") as fp: + self.ssh_pubkey = fp.read().rstrip('\n') + + if not meta_data.get('public-keys'): + meta_data['public-keys'] = [] + meta_data['public-keys'].append(self.ssh_pubkey) + self.ssh_port = None self.pid = None self.pid_file = None self.console_file = None self.disk = image_path + self.meta_data = meta_data super(NoCloudKVMInstance, self).__init__( platform, name, properties, config, features) @@ -78,11 +105,15 @@ class NoCloudKVMInstance(Instance): """Generate nocloud seed from user-data""" seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) + meta_data_file = os.path.join(tmpdir, '%s_meta_data' % self.name) with open(user_data_file, "w") as ud_file: ud_file.write(self.user_data) - c_util.subp(['cloud-localds', seed_file, user_data_file]) + # meta-data can be yaml, but more easily pretty printed with json + write_json(meta_data_file, self.meta_data) + c_util.subp(['cloud-localds', seed_file, user_data_file, + meta_data_file]) return seed_file diff --git a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py index 0005e1f2..2dae3590 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py +++ b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py @@ -41,10 +41,6 @@ class NoCloudKVMSnapshot(Snapshot): @param use_desc: description of snapshot instance use @return_value: an Instance """ - key_file = os.path.join(self.platform.config['data_dir'], - self.platform.config['public_key']) - user_data = self.inject_ssh_key(user_data, key_file) - instance = self.platform.create_instance( self.properties, self.config, self.features, self._image_path, image_desc=str(self), use_desc=use_desc, @@ -55,22 +51,6 @@ class NoCloudKVMSnapshot(Snapshot): return instance - def inject_ssh_key(self, user_data, key_file): - """Inject the authorized key into the user_data.""" - with open(key_file) as f: - value = f.read() - - key = 'ssh_authorized_keys:' - value = ' - %s' % value.strip() - user_data = user_data.split('\n') - if key in user_data: - user_data.insert(user_data.index(key) + 1, '%s' % value) - else: - user_data.insert(-1, '%s' % key) - user_data.insert(-1, '%s' % value) - - return '\n'.join(user_data) - def destroy(self): """Clean up snapshot data.""" shutil.rmtree(self._workd) -- cgit v1.2.3 From b63ee73da874de68ff2019570e12df2a39d4626b Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 12 Dec 2017 11:28:05 -0700 Subject: tests: fix collect_console when not implemented The exception was incorrectly creating a string and not a bytes object. --- tests/cloud_tests/collect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 4805cea1..bb722457 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -31,8 +31,8 @@ def collect_console(instance, base_dir): LOG.debug('getting console log') try: data = instance.console_log() - except NotImplementedError as e: - data = 'Not Implemented: %s' % e + except NotImplementedError: + data = b'instance.console_log: not implemented' with open(os.path.join(base_dir, 'console.log'), "wb") as fp: fp.write(data) -- cgit v1.2.3 From 34595e9b4abacc10ac599aad97c95861af34ea54 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 7 Dec 2017 12:54:46 -0800 Subject: tests: Enable AWS EC2 Integration Testing This enables integration tests to utilize AWS EC2 as a testing platform by utilizing the boto3 Python library. Usage will create and delete a custom VPC for every run. All resources will be tagged with the ec2 tag, 'cii', and the date (e.g. cii-20171220-102452). The VPC is setup with both IPv4 and IPv6 capabilities, but will only hand out IPv4 addresses by default. Instances will have complete Internet access and have full ingress and egress access (i.e. no firewall). SSH keys are generated with each run of the integration tests with the key getting uploaded to AWS at the start of tests and deleted on exit. To enable creation when the platform is setup the SSH generation code is moved to be completed by the platform setup and not during image setup. The nocloud-kvm platform was updated with this change. Creating a custom image will utilize the same clean script, boot_clean_script, that the LXD platform uses as well. The custom AMI is generated, used, and de-registered after a test run. The default instance type is set to t2.micro. This is one of the smallest instance types and is free tier eligible. The default timeout for ec2 was increased to 300 from 120 as many tests hit up against the 2 minute timeout and depending on region load can go over. Documentation for the AWS platform was added with the expected configuration files for the platform to be used. There are some additional whitespace changes included as well. pylint exception was added for paramiko and simplestreams. In the past these were not already flagged due to no __init__.py in the subdirectories of files that used these. boto3 was added to the list of dependencies in the tox ci-test runner. In order to grab console logs on EC2 the harness will now shut down an instance before terminating and before collecting the console log. This is to address a behavior of EC2 where the console log is refreshed very infrequently, but one point when it is refreshed is after shutdown. --- .pylintrc | 2 +- doc/rtd/topics/tests.rst | 38 +++- tests/cloud_tests/collect.py | 19 +- tests/cloud_tests/platforms.yaml | 11 +- tests/cloud_tests/platforms/__init__.py | 2 + tests/cloud_tests/platforms/ec2/image.py | 109 ++++++++++ tests/cloud_tests/platforms/ec2/instance.py | 126 +++++++++++ tests/cloud_tests/platforms/ec2/platform.py | 231 +++++++++++++++++++++ tests/cloud_tests/platforms/ec2/snapshot.py | 66 ++++++ tests/cloud_tests/platforms/instances.py | 70 ++++++- tests/cloud_tests/platforms/nocloudkvm/instance.py | 88 ++++---- tests/cloud_tests/platforms/nocloudkvm/platform.py | 4 + tests/cloud_tests/platforms/platforms.py | 69 ++++++ tests/cloud_tests/releases.yaml | 8 +- tests/cloud_tests/setup_image.py | 18 -- tests/cloud_tests/util.py | 17 +- tox.ini | 1 + 17 files changed, 784 insertions(+), 95 deletions(-) create mode 100644 tests/cloud_tests/platforms/ec2/image.py create mode 100644 tests/cloud_tests/platforms/ec2/instance.py create mode 100644 tests/cloud_tests/platforms/ec2/platform.py create mode 100644 tests/cloud_tests/platforms/ec2/snapshot.py (limited to 'tests/cloud_tests') diff --git a/.pylintrc b/.pylintrc index 3ad36924..05a086d9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -46,7 +46,7 @@ reports=no # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=six.moves,pkg_resources,httplib,http.client +ignored-modules=six.moves,pkg_resources,httplib,http.client,paramiko,simplestreams # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index d668e3f4..bf04bb3c 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -118,19 +118,19 @@ TreeRun and TreeCollect If working on a cloud-init feature or resolving a bug, it may be useful to run the current copy of cloud-init in the integration testing environment. -The integration testing suite can automatically build a deb based on the +The integration testing suite can automatically build a deb based on the current working tree of cloud-init and run the test suite using this deb. The ``tree_run`` and ``tree_collect`` commands take the same arguments as -the ``run`` and ``collect`` commands. These commands will build a deb and -write it into a temporary file, then start the test suite and pass that deb +the ``run`` and ``collect`` commands. These commands will build a deb and +write it into a temporary file, then start the test suite and pass that deb in. To build a deb only, and not run the test suite, the ``bddeb`` command can be used. Note that code in the cloud-init working tree that has not been committed when the cloud-init deb is built will still be included. To build a cloud-init deb from or use the ``tree_run`` command using a copy of -cloud-init located in a different directory, use the option ``--cloud-init +cloud-init located in a different directory, use the option ``--cloud-init /path/to/cloud-init``. .. code-block:: bash @@ -383,7 +383,7 @@ Development Checklist * Valid unit tests validating output collected * Passes pylint & pep8 checks * Placed in the appropriate sub-folder in the test cases directory -* Tested by running the test: +* Tested by running the test: .. code-block:: bash @@ -392,6 +392,32 @@ Development Checklist --test modules/your_test.yaml \ [--deb ] + +Platforms +========= + +EC2 +--- +To run on the EC2 platform it is required that the user has an AWS credentials +configuration file specifying his or her access keys and a default region. +These configuration files are the standard that the AWS cli and other AWS +tools utilize for interacting directly with AWS itself and are normally +generated when running ``aws configure``: + +.. code-block:: bash + + $ cat $HOME/.aws/credentials + [default] + aws_access_key_id = + aws_secret_access_key = + +.. code-block:: bash + + $ cat $HOME/.aws/config + [default] + region = us-west-2 + + Architecture ============ @@ -455,7 +481,7 @@ replace the default. If the data is a dictionary then the value will be the result of merging that dictionary from the default config and that dictionary from the overrides. -Merging is done using the function +Merging is done using the function ``tests.cloud_tests.config.merge_config``, which can be examined for more detail on config merging behavior. diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index bb722457..33acbb1e 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -28,12 +28,18 @@ def collect_script(instance, base_dir, script, script_name): def collect_console(instance, base_dir): - LOG.debug('getting console log') + """Collect instance console log. + + @param instance: instance to get console log for + @param base_dir: directory to write console log to + """ + logfile = os.path.join(base_dir, 'console.log') + LOG.debug('getting console log for %s to %s', instance, logfile) try: data = instance.console_log() except NotImplementedError: data = b'instance.console_log: not implemented' - with open(os.path.join(base_dir, 'console.log'), "wb") as fp: + with open(logfile, "wb") as fp: fp.write(data) @@ -89,12 +95,11 @@ def collect_test_data(args, snapshot, os_name, test_name): test_output_dir, script, script_name)) for script_name, script in test_scripts.items()] - console_log = partial( - run_single, 'collect console', - partial(collect_console, instance, test_output_dir)) - res = run_stage('collect for test: {}'.format(test_name), - [start_call] + collect_calls + [console_log]) + [start_call] + collect_calls) + + instance.shutdown() + collect_console(instance, test_output_dir) return res diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index fa4f845e..cb1c904b 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -6,8 +6,13 @@ default_platform_config: get_image_timeout: 300 # maximum time to create instance (before waiting for cloud-init) create_instance_timeout: 60 - + private_key: id_rsa + public_key: id_rsa.pub platforms: + ec2: + enabled: true + instance-type: t2.micro + tag: cii lxd: enabled: true # overrides for image templates @@ -61,9 +66,5 @@ platforms: {{ config_get("user.vendor-data", properties.default) }} nocloud-kvm: enabled: true - private_key: id_rsa - public_key: id_rsa.pub - ec2: {} - azure: {} # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index 92ed1627..a01e51ac 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -2,10 +2,12 @@ """Main init.""" +from .ec2 import platform as ec2 from .lxd import platform as lxd from .nocloudkvm import platform as nocloudkvm PLATFORMS = { + 'ec2': ec2.EC2Platform, 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, 'lxd': lxd.LXDPlatform, } diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py new file mode 100644 index 00000000..53706b1d --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -0,0 +1,109 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""EC2 Image Base Class.""" + +from ..images import Image +from .snapshot import EC2Snapshot +from tests.cloud_tests import LOG + + +class EC2Image(Image): + """EC2 backed image.""" + + platform_name = 'ec2' + + def __init__(self, platform, config, image_ami): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param image_ami: string of image ami ID + """ + super(EC2Image, self).__init__(platform, config) + self._img_instance = None + self.image_ami = image_ami + + @property + def _instance(self): + """Internal use only, returns a running instance""" + if not self._img_instance: + self._img_instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_ami, user_data=None) + self._img_instance.start(wait=True, wait_for_cloud_init=True) + return self._img_instance + + @property + def properties(self): + """Dictionary containing: 'arch', 'os', 'version', 'release'.""" + return { + 'arch': self.config['arch'], + 'os': self.config['family'], + 'release': self.config['release'], + 'version': self.config['version'], + } + + def destroy(self): + """Delete the instance used to create a custom image.""" + if self._img_instance: + LOG.debug('terminating backing instance %s', + self._img_instance.instance.instance_id) + self._img_instance.instance.terminate() + self._img_instance.instance.wait_until_terminated() + + super(EC2Image, self).destroy() + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + self._instance.start(wait=True) + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + self._instance.start(wait=True) + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + self._instance.start(wait=True) + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done. + + Will return base image_ami if no instance has been booted, otherwise + will run the clean script, shutdown the instance, create a custom + AMI, and use that AMI once available. + """ + if not self._img_instance: + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, self.image_ami, + delete_on_destroy=False) + + if self.config.get('boot_clean_script'): + self._img_instance.run_script(self.config.get('boot_clean_script')) + + self._img_instance.shutdown(wait=True) + + LOG.debug('creating custom ami from instance %s', + self._img_instance.instance.instance_id) + response = self.platform.ec2_client.create_image( + Name='%s-%s' % (self.platform.tag, self.image_ami), + InstanceId=self._img_instance.instance.instance_id + ) + image_ami_edited = response['ImageId'] + + # Create image and wait until it is in the 'available' state + image = self.platform.ec2_resource.Image(image_ami_edited) + image.wait_until_exists() + waiter = self.platform.ec2_client.get_waiter('image_available') + waiter.wait(ImageIds=[image.id]) + image.reload() + + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, image_ami_edited) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py new file mode 100644 index 00000000..4ba737ab --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -0,0 +1,126 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 instance.""" +import os + +import botocore + +from ..instances import Instance +from tests.cloud_tests import LOG, util + + +class EC2Instance(Instance): + """EC2 backed instance.""" + + platform_name = "ec2" + _ssh_client = None + + def __init__(self, platform, properties, config, features, + image_ami, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_ami: AWS AMI ID for image to use + @param user_data: test user-data to pass to instance + """ + super(EC2Instance, self).__init__( + platform, image_ami, properties, config, features) + + self.image_ami = image_ami + self.instance = None + self.user_data = user_data + self.ssh_ip = None + self.ssh_port = 22 + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + + def console_log(self): + """Collect console log from instance. + + The console log is buffered and not always present, therefore + may return empty string. + """ + try: + return self.instance.console_output()['Output'].encode() + except KeyError: + return b'' + + def destroy(self): + """Clean up instance.""" + if self.instance: + LOG.debug('destroying instance %s', self.instance.id) + self.instance.terminate() + self.instance.wait_until_terminated() + + self._ssh_close() + + super(EC2Instance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance on EC2 with the platfrom's VPC.""" + if self.instance: + if self.instance.state['Name'] == 'running': + return + + LOG.debug('starting instance %s', self.instance.id) + self.instance.start() + else: + LOG.debug('launching instance') + + args = { + 'ImageId': self.image_ami, + 'InstanceType': self.platform.instance_type, + 'KeyName': self.platform.key_name, + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': [self.platform.security_group.id], + 'SubnetId': self.platform.subnet.id, + 'TagSpecifications': [{ + 'ResourceType': 'instance', + 'Tags': [{ + 'Key': 'Name', 'Value': self.platform.tag + }] + }], + } + + if self.user_data: + args['UserData'] = self.user_data + + try: + instances = self.platform.ec2_resource.create_instances(**args) + except botocore.exceptions.ClientError as error: + error_msg = error.response['Error']['Message'] + raise util.PlatformError('start', error_msg) + + self.instance = instances[0] + + LOG.debug('instance id: %s', self.instance.id) + if wait: + self.instance.wait_until_running() + self.instance.reload() + self.ssh_ip = self.instance.public_ip_address + self._wait_for_system(wait_for_cloud_init) + + def shutdown(self, wait=True): + """Shutdown instance.""" + LOG.debug('stopping instance %s', self.instance.id) + self.instance.stop() + + if wait: + self.instance.wait_until_stopped() + self.instance.reload() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py new file mode 100644 index 00000000..fdb17ba0 --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -0,0 +1,231 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 platform.""" +from datetime import datetime +import os + +import boto3 +import botocore + +from ..platforms import Platform +from .image import EC2Image +from .instance import EC2Instance +from tests.cloud_tests import LOG + + +class EC2Platform(Platform): + """EC2 test platform.""" + + platform_name = 'ec2' + ipv4_cidr = '192.168.1.0/20' + + def __init__(self, config): + """Set up platform.""" + super(EC2Platform, self).__init__(config) + # Used for unique VPC, SSH key, and custom AMI generation naming + self.tag = '%s-%s' % ( + config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) + self.instance_type = config['instance-type'] + + try: + self.ec2_client = boto3.client('ec2') + self.ec2_resource = boto3.resource('ec2') + self.ec2_region = boto3.Session().region_name + self.key_name = self._upload_public_key(config) + except botocore.exceptions.NoRegionError: + raise RuntimeError( + 'Please configure default region in $HOME/.aws/config') + except botocore.exceptions.NoCredentialsError: + raise RuntimeError( + 'Please configure ec2 credentials in $HOME/.aws/credentials') + + self.vpc = self._create_vpc() + self.internet_gateway = self._create_internet_gateway() + self.subnet = self._create_subnet() + self.routing_table = self._create_routing_table() + self.security_group = self._create_security_group() + + def create_instance(self, properties, config, features, + image_ami, user_data=None): + """Create an instance + + @param src_img_path: image path to launch from + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_ami: string of image ami ID + @param user_data: test user-data to pass to instance + @return_value: cloud_tests.instances instance + """ + return EC2Instance(self, properties, config, features, + image_ami, user_data) + + def destroy(self): + """Delete SSH keys, terminate all instances, and delete VPC.""" + for instance in self.vpc.instances.all(): + LOG.debug('waiting for instance %s termination', instance.id) + instance.terminate() + instance.wait_until_terminated() + + if self.key_name: + LOG.debug('deleting SSH key %s', self.key_name) + self.ec2_client.delete_key_pair(KeyName=self.key_name) + + if self.security_group: + LOG.debug('deleting security group %s', self.security_group.id) + self.security_group.delete() + + if self.subnet: + LOG.debug('deleting subnet %s', self.subnet.id) + self.subnet.delete() + + if self.routing_table: + LOG.debug('deleting routing table %s', self.routing_table.id) + self.routing_table.delete() + + if self.internet_gateway: + LOG.debug('deleting internet gateway %s', self.internet_gateway.id) + self.internet_gateway.detach_from_vpc(VpcId=self.vpc.id) + self.internet_gateway.delete() + + if self.vpc: + LOG.debug('deleting vpc %s', self.vpc.id) + self.vpc.delete() + + def get_image(self, img_conf): + """Get image using specified image configuration. + + Hard coded for 'amd64' based images. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + if img_conf['root-store'] == 'ebs': + root_store = 'ssd' + elif img_conf['root-store'] == 'instance-store': + root_store = 'instance' + else: + raise RuntimeError('Unknown root-store type: %s' % + (img_conf['root-store'])) + + filters = [ + 'arch=%s' % 'amd64', + 'endpoint=https://ec2.%s.amazonaws.com' % self.ec2_region, + 'region=%s' % self.ec2_region, + 'release=%s' % img_conf['release'], + 'root_store=%s' % root_store, + 'virt=hvm', + ] + + LOG.debug('finding image using streams') + image = self._query_streams(img_conf, filters) + + try: + image_ami = image['id'] + except KeyError: + raise RuntimeError('No images found for %s!' % img_conf['release']) + + LOG.debug('found image: %s', image_ami) + image = EC2Image(self, img_conf, image_ami) + return image + + def _create_internet_gateway(self): + """Create Internet Gateway and assign to VPC.""" + LOG.debug('creating internet gateway') + internet_gateway = self.ec2_resource.create_internet_gateway() + internet_gateway.attach_to_vpc(VpcId=self.vpc.id) + self._tag_resource(internet_gateway) + + return internet_gateway + + def _create_routing_table(self): + """Update default routing table with internet gateway. + + This sets up internet access between the VPC via the internet gateway + by configuring routing tables for IPv4 and IPv6. + """ + LOG.debug('creating routing table') + route_table = self.vpc.create_route_table() + route_table.create_route(DestinationCidrBlock='0.0.0.0/0', + GatewayId=self.internet_gateway.id) + route_table.create_route(DestinationIpv6CidrBlock='::/0', + GatewayId=self.internet_gateway.id) + route_table.associate_with_subnet(SubnetId=self.subnet.id) + self._tag_resource(route_table) + + return route_table + + def _create_security_group(self): + """Enables ingress to default VPC security group.""" + LOG.debug('creating security group') + security_group = self.vpc.create_security_group( + GroupName=self.tag, Description='integration test security group') + security_group.authorize_ingress( + IpProtocol='-1', FromPort=-1, ToPort=-1, CidrIp='0.0.0.0/0') + self._tag_resource(security_group) + + return security_group + + def _create_subnet(self): + """Generate IPv4 and IPv6 subnets for use.""" + ipv6_cidr = self.vpc.ipv6_cidr_block_association_set[0][ + 'Ipv6CidrBlock'][:-2] + '64' + + LOG.debug('creating subnet with following ranges:') + LOG.debug('ipv4: %s', self.ipv4_cidr) + LOG.debug('ipv6: %s', ipv6_cidr) + subnet = self.vpc.create_subnet(CidrBlock=self.ipv4_cidr, + Ipv6CidrBlock=ipv6_cidr) + modify_subnet = subnet.meta.client.modify_subnet_attribute + modify_subnet(SubnetId=subnet.id, + MapPublicIpOnLaunch={'Value': True}) + self._tag_resource(subnet) + + return subnet + + def _create_vpc(self): + """Setup AWS EC2 VPC or return existing VPC.""" + LOG.debug('creating new vpc') + try: + vpc = self.ec2_resource.create_vpc( + CidrBlock=self.ipv4_cidr, + AmazonProvidedIpv6CidrBlock=True) + except botocore.exceptions.ClientError as e: + raise RuntimeError(e) + + vpc.wait_until_available() + self._tag_resource(vpc) + + return vpc + + def _tag_resource(self, resource): + """Tag a resource with the specified tag. + + This makes finding and deleting resources specific to this testing + much easier to find. + + @param resource: resource to tag + """ + tag = { + 'Key': 'Name', + 'Value': self.tag + } + resource.create_tags(Tags=[tag]) + + def _upload_public_key(self, config): + """Generate random name and upload SSH key with that name. + + @param config: platform config + @return: string of ssh key name + """ + key_file = os.path.join(config['data_dir'], config['public_key']) + with open(key_file, 'r') as file: + public_key = file.read().strip('\n') + + LOG.debug('uploading SSH key %s', self.tag) + self.ec2_client.import_key_pair(KeyName=self.tag, + PublicKeyMaterial=public_key) + + return self.tag + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/snapshot.py b/tests/cloud_tests/platforms/ec2/snapshot.py new file mode 100644 index 00000000..2c48cb54 --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/snapshot.py @@ -0,0 +1,66 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 snapshot.""" + +from ..snapshots import Snapshot +from tests.cloud_tests import LOG + + +class EC2Snapshot(Snapshot): + """EC2 image copy backed snapshot.""" + + platform_name = 'ec2' + + def __init__(self, platform, properties, config, features, image_ami, + delete_on_destroy=True): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + @param image_ami: string of image ami ID + @param delete_on_destroy: boolean to delete on destroy + """ + super(EC2Snapshot, self).__init__( + platform, properties, config, features) + + self.image_ami = image_ami + self.delete_on_destroy = delete_on_destroy + + def destroy(self): + """Deregister the backing AMI.""" + if self.delete_on_destroy: + image = self.platform.ec2_resource.Image(self.image_ami) + snapshot_id = image.block_device_mappings[0]['Ebs']['SnapshotId'] + + LOG.debug('removing custom ami %s', self.image_ami) + self.platform.ec2_client.deregister_image(ImageId=self.image_ami) + + LOG.debug('removing custom snapshot %s', snapshot_id) + self.platform.ec2_client.delete_snapshot(SnapshotId=snapshot_id) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param meta_data: meta_data for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: string of test name + @return_value: an Instance + """ + if meta_data is not None: + raise ValueError("metadata not supported on Ec2") + + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_ami, user_data) + + if start: + instance.start() + + return instance + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index 8c59d62c..3bad021f 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -1,14 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. """Base instance.""" +import time + +import paramiko +from paramiko.ssh_exception import ( + BadHostKeyException, AuthenticationException, SSHException) from ..util import TargetBase +from tests.cloud_tests import LOG, util class Instance(TargetBase): """Base instance object.""" platform_name = None + _ssh_client = None def __init__(self, platform, name, properties, config, features): """Set up instance. @@ -26,6 +33,11 @@ class Instance(TargetBase): self.features = features self._tmp_count = 0 + self.ssh_ip = None + self.ssh_port = None + self.ssh_key_file = None + self.ssh_username = 'ubuntu' + def console_log(self): """Instance console. @@ -47,7 +59,63 @@ class Instance(TargetBase): def destroy(self): """Clean up instance.""" - pass + self._ssh_close() + + def _ssh(self, command, stdin=None): + """Run a command via SSH.""" + client = self._ssh_connect() + + cmd = util.shell_pack(command) + fp_in, fp_out, fp_err = client.exec_command(cmd) + channel = fp_in.channel + + if stdin is not None: + fp_in.write(stdin) + fp_in.close() + + channel.shutdown_write() + rc = channel.recv_exit_status() + + return (fp_out.read(), fp_err.read(), rc) + + def _ssh_close(self): + if self._ssh_client: + try: + self._ssh_client.close() + except SSHException: + LOG.warning('Failed to close SSH connection.') + self._ssh_client = None + + def _ssh_connect(self): + """Connect via SSH.""" + if self._ssh_client: + return self._ssh_client + + if not self.ssh_ip or not self.ssh_port: + raise ValueError + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) + + retries = 30 + while retries: + try: + client.connect(username=self.ssh_username, + hostname=self.ssh_ip, port=self.ssh_port, + pkey=private_key, banner_timeout=30) + self._ssh_client = client + return client + except (ConnectionRefusedError, AuthenticationException, + BadHostKeyException, ConnectionResetError, SSHException, + OSError) as e: + retries -= 1 + time.sleep(10) + + ssh_cmd = 'Failed ssh connection to %s@%s:%s after 300 seconds' % ( + self.ssh_username, self.ssh_ip, self.ssh_port + ) + raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') def _wait_for_system(self, wait_for_cloud_init): """Wait until system has fully booted and cloud-init has finished. diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index 9bb24256..932dc0fa 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -4,7 +4,6 @@ import copy import os -import paramiko import socket import subprocess import time @@ -13,7 +12,7 @@ import uuid from ..instances import Instance from cloudinit.atomic_helper import write_json from cloudinit import util as c_util -from tests.cloud_tests import util +from tests.cloud_tests import LOG, util # This domain contains reverse lookups for hostnames that are used. # The primary reason is so sudo will return quickly when it attempts @@ -26,7 +25,6 @@ class NoCloudKVMInstance(Instance): """NoCloud KVM backed instance.""" platform_name = "nocloud-kvm" - _ssh_client = None def __init__(self, platform, name, image_path, properties, config, features, user_data, meta_data): @@ -39,6 +37,10 @@ class NoCloudKVMInstance(Instance): @param config: dictionary of configuration values @param features: dictionary of supported feature flags """ + super(NoCloudKVMInstance, self).__init__( + platform, name, properties, config, features + ) + self.user_data = user_data if meta_data: meta_data = copy.deepcopy(meta_data) @@ -66,6 +68,7 @@ class NoCloudKVMInstance(Instance): meta_data['public-keys'] = [] meta_data['public-keys'].append(self.ssh_pubkey) + self.ssh_ip = '127.0.0.1' self.ssh_port = None self.pid = None self.pid_file = None @@ -73,8 +76,33 @@ class NoCloudKVMInstance(Instance): self.disk = image_path self.meta_data = meta_data - super(NoCloudKVMInstance, self).__init__( - platform, name, properties, config, features) + def shutdown(self, wait=True): + """Shutdown instance.""" + + if self.pid: + # This relies on _execute which uses sudo over ssh. The ssh + # connection would get killed before sudo exited, so ignore errors. + cmd = ['shutdown', 'now'] + try: + self._execute(cmd) + except util.InTargetExecuteError: + pass + self._ssh_close() + + if wait: + LOG.debug("Executed shutdown. waiting on pid %s to end", + self.pid) + time_for_shutdown = 120 + give_up_at = time.time() + time_for_shutdown + pid_file_path = '/proc/%s' % self.pid + msg = ("pid %s did not exit in %s seconds after shutdown." % + (self.pid, time_for_shutdown)) + while True: + if not os.path.exists(pid_file_path): + break + if time.time() > give_up_at: + raise util.PlatformError("shutdown", msg) + self.pid = None def destroy(self): """Clean up instance.""" @@ -88,9 +116,7 @@ class NoCloudKVMInstance(Instance): os.remove(self.pid_file) self.pid = None - if self._ssh_client: - self._ssh_client.close() - self._ssh_client = None + self._ssh_close() super(NoCloudKVMInstance, self).destroy() @@ -99,7 +125,7 @@ class NoCloudKVMInstance(Instance): if env: env_args = ['env'] + ["%s=%s" for k, v in env.items()] - return self.ssh(['sudo'] + env_args + list(command), stdin=stdin) + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) def generate_seed(self, tmpdir): """Generate nocloud seed from user-data""" @@ -125,50 +151,6 @@ class NoCloudKVMInstance(Instance): s.close() return num - def ssh(self, command, stdin=None): - """Run a command via SSH.""" - client = self._ssh_connect() - - cmd = util.shell_pack(command) - try: - fp_in, fp_out, fp_err = client.exec_command(cmd) - channel = fp_in.channel - if stdin is not None: - fp_in.write(stdin) - fp_in.close() - - channel.shutdown_write() - rc = channel.recv_exit_status() - return (fp_out.read(), fp_err.read(), rc) - except paramiko.SSHException as e: - raise util.InTargetExecuteError( - b'', b'', -1, command, self.name, reason=e) - - def _ssh_connect(self, hostname='localhost', username='ubuntu', - banner_timeout=120, retry_attempts=30): - """Connect via SSH.""" - if self._ssh_client: - return self._ssh_client - - private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - while retry_attempts: - try: - client.connect(hostname=hostname, username=username, - port=self.ssh_port, pkey=private_key, - banner_timeout=banner_timeout) - self._ssh_client = client - return client - except (paramiko.SSHException, TypeError): - time.sleep(1) - retry_attempts = retry_attempts - 1 - - error_desc = 'Failed command to: %s@%s:%s' % (username, hostname, - self.ssh_port) - raise util.InTargetExecuteError('', '', -1, 'ssh connect', - self.name, error_desc) - def start(self, wait=True, wait_for_cloud_init=False): """Start instance.""" tmpdir = self.platform.config['data_dir'] diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 85933463..a7e6f5de 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -21,6 +21,10 @@ class NoCloudKVMPlatform(Platform): platform_name = 'nocloud-kvm' + def __init__(self, config): + """Set up platform.""" + super(NoCloudKVMPlatform, self).__init__(config) + def get_image(self, img_conf): """Get image using specified image configuration. diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 28975368..d4e5c561 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -1,6 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. """Base platform class.""" +import os + +from simplestreams import filters, mirrors +from simplestreams import util as s_util + +from cloudinit import util as c_util class Platform(object): @@ -11,6 +17,7 @@ class Platform(object): def __init__(self, config): """Set up platform.""" self.config = config + self._generate_ssh_keys(config['data_dir']) def get_image(self, img_conf): """Get image using specified image configuration. @@ -24,4 +31,66 @@ class Platform(object): """Clean up platform data.""" pass + def _generate_ssh_keys(self, data_dir): + """Generate SSH keys to be used with image.""" + filename = os.path.join(data_dir, 'id_rsa') + + if os.path.exists(filename): + c_util.del_file(filename) + + c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096', + '-f', filename, '-P', '', + '-C', 'ubuntu@cloud_test'], + capture=True) + + @staticmethod + def _query_streams(img_conf, img_filter): + """Query streams for latest image given a specific filter. + + @param img_conf: configuration for image + @param filters: array of filters as strings format 'key=value' + @return: dictionary with latest image information or empty + """ + def policy(content, path): + return s_util.read_signed(content, keyring=img_conf['keyring']) + + (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) + smirror = mirrors.UrlMirrorReader(url, policy=policy) + + config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} + tmirror = FilterMirror(config) + tmirror.sync(smirror, path) + + try: + return tmirror.json_entries[0] + except IndexError: + raise RuntimeError('no images found with filter: %s' % img_filter) + + +class FilterMirror(mirrors.BasicMirrorWriter): + """Taken from sstream-query to return query result as json array.""" + + def __init__(self, config=None): + super(FilterMirror, self).__init__(config=config) + if config is None: + config = {} + self.config = config + self.filters = config.get('filters', []) + self.json_entries = [] + + def load_products(self, path=None, content_id=None): + return {'content_id': content_id, 'products': {}} + + def filter_item(self, data, src, target, pedigree): + return filters.filter_item(self.filters, data, src, pedigree) + + def insert_item(self, data, src, target, pedigree, contentsource): + # src and target are top level products:1.0 + # data is src['products'][ped[0]]['versions'][ped[1]]['items'][ped[2]] + # contentsource is a ContentSource if 'path' exists in data or None + data = s_util.products_exdata(src, pedigree) + if 'path' in data: + data.update({'item_url': contentsource.url}) + self.json_entries.append(data) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index e5933802..48f903b8 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -27,10 +27,14 @@ default_release_config: # features groups and additional feature settings feature_groups: [] features: {} - nocloud-kvm: mirror_url: https://cloud-images.ubuntu.com/daily - mirror_dir: '/srv/citest/nocloud-kvm' + mirror_dir: '/srv/citest/images' keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg + ec2: + # Choose from: [ebs, instance-store] + root-store: ebs + boot_timeout: 300 + nocloud-kvm: setup_overrides: null override_templates: false # lxd specific default configuration options diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 179f40db..6d242115 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -5,7 +5,6 @@ from functools import partial import os -from cloudinit import util as c_util from tests.cloud_tests import LOG from tests.cloud_tests import stage, util @@ -192,20 +191,6 @@ def enable_repo(args, image): image.execute(cmd, description=msg) -def generate_ssh_keys(data_dir): - """Generate SSH keys to be used with image.""" - LOG.info('generating SSH keys') - filename = os.path.join(data_dir, 'id_rsa') - - if os.path.exists(filename): - c_util.del_file(filename) - - c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096', - '-f', filename, '-P', '', - '-C', 'ubuntu@cloud_test'], - capture=True) - - def setup_image(args, image): """Set up image as specified in args. @@ -239,9 +224,6 @@ def setup_image(args, image): LOG.info('setting up %s', image) res = stage.run_stage( 'set up for {}'.format(image), calls, continue_after_error=False) - LOG.debug('after setup complete, installed cloud-init version is: %s', - installed_package_version(image, 'cloud-init')) - generate_ssh_keys(args.data_dir) return res # vi: ts=4 expandtab diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 2aedcd0d..6ff285e7 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -321,9 +321,9 @@ class TargetBase(object): rcs = (0,) if description: - LOG.debug('Executing "%s"', description) + LOG.debug('executing "%s"', description) else: - LOG.debug("Executing command: %s", shell_quote(command)) + LOG.debug("executing command: %s", shell_quote(command)) out, err, rc = self._execute(command=command, stdin=stdin, env=env) @@ -447,6 +447,19 @@ class InTargetExecuteError(c_util.ProcessExecutionError): reason=reason) +class PlatformError(IOError): + """Error type for platform errors.""" + + default_desc = 'unexpected error in platform.' + + def __init__(self, operation, description=None): + """Init error and parent error class.""" + description = description if description else self.default_desc + + message = '%s: %s' % (operation, description) + IOError.__init__(self, message) + + class TempDir(object): """Configurable temporary directory like tempfile.TemporaryDirectory.""" diff --git a/tox.ini b/tox.ini index fdc8a665..88b82dc3 100644 --- a/tox.ini +++ b/tox.ini @@ -134,4 +134,5 @@ passenv = HOME deps = pylxd==2.2.4 paramiko==2.3.1 + boto3==1.4.8 bzr+lp:simplestreams -- cgit v1.2.3 From f794708fabba690677e0e81bd929871c83af3409 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 8 Jan 2018 08:53:14 -0800 Subject: tests: rename test ssh keys to avoid appearance of leaking private keys. While the generated ssh keys are throw away keys, generating SSH keys with the default name (e.g. id_rsa) can trigger security scanners or draw unnecessary attention. The change here simply renames 'id_rsa' to 'cloud_init_rsa' to avoid a false positive reported by a scanning tool. --- tests/cloud_tests/platforms.yaml | 4 ++-- tests/cloud_tests/platforms/platforms.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index cb1c904b..448aa98d 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -6,8 +6,8 @@ default_platform_config: get_image_timeout: 300 # maximum time to create instance (before waiting for cloud-init) create_instance_timeout: 60 - private_key: id_rsa - public_key: id_rsa.pub + private_key: cloud_init_rsa + public_key: cloud_init_rsa.pub platforms: ec2: enabled: true diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index d4e5c561..1542b3be 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -33,7 +33,7 @@ class Platform(object): def _generate_ssh_keys(self, data_dir): """Generate SSH keys to be used with image.""" - filename = os.path.join(data_dir, 'id_rsa') + filename = os.path.join(data_dir, self.config['private_key']) if os.path.exists(filename): c_util.del_file(filename) -- cgit v1.2.3 From 72270e8c311efc8b9ba8bb92492d8728d84bd9f2 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 8 Jan 2018 10:00:35 -0800 Subject: tests: clean up image properties This fixes the incorrectly named 'family' value for images as 'os'. Families are already defined in util.py:OS_FAMILY_MAPPING and a family is a collection of OSes. This makes the properties function part of the super class of image as it is only overrided by the lxd backend. --- tests/cloud_tests/platforms/ec2/image.py | 10 ---------- tests/cloud_tests/platforms/images.py | 3 ++- tests/cloud_tests/platforms/nocloudkvm/image.py | 10 ---------- tests/cloud_tests/releases.yaml | 10 +++++----- 4 files changed, 7 insertions(+), 26 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py index 53706b1d..7bedf59d 100644 --- a/tests/cloud_tests/platforms/ec2/image.py +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -33,16 +33,6 @@ class EC2Image(Image): self._img_instance.start(wait=True, wait_for_cloud_init=True) return self._img_instance - @property - def properties(self): - """Dictionary containing: 'arch', 'os', 'version', 'release'.""" - return { - 'arch': self.config['arch'], - 'os': self.config['family'], - 'release': self.config['release'], - 'version': self.config['version'], - } - def destroy(self): """Delete the instance used to create a custom image.""" if self._img_instance: diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py index d503108a..557a5cf6 100644 --- a/tests/cloud_tests/platforms/images.py +++ b/tests/cloud_tests/platforms/images.py @@ -26,7 +26,8 @@ class Image(TargetBase): @property def properties(self): """{} containing: 'arch', 'os', 'version', 'release'.""" - raise NotImplementedError + return {k: self.config[k] + for k in ('arch', 'os', 'release', 'version')} @property def features(self): diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py index 09ff2a3b..bc2b6e75 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/image.py +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -35,16 +35,6 @@ class NoCloudKVMImage(Image): super(NoCloudKVMImage, self).__init__(platform, config) - @property - def properties(self): - """Dictionary containing: 'arch', 'os', 'version', 'release'.""" - return { - 'arch': self.config['arch'], - 'os': self.config['family'], - 'release': self.config['release'], - 'version': self.config['version'], - } - def _execute(self, command, stdin=None, env=None): """Execute command in image, modifying image.""" return self.mount_image_callback(command, stdin=stdin, env=env) diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 48f903b8..0a9fa602 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -132,7 +132,7 @@ releases: enabled: true release: bionic version: 18.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -148,7 +148,7 @@ releases: enabled: true release: artful version: 17.10 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -164,7 +164,7 @@ releases: enabled: true release: zesty version: 17.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -180,7 +180,7 @@ releases: enabled: true release: xenial version: 16.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -196,7 +196,7 @@ releases: enabled: true release: trusty version: 14.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base -- cgit v1.2.3 From df24daa833d7eb88e7c172eb5d7f257766adb0e3 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 8 Jan 2018 09:43:44 -0800 Subject: tests: update apt sources list test Due to additional platforms getting added this test was not taking into account platform specific mirrors nor was it checking that no additional entries were added. --- tests/cloud_tests/testcases/modules/apt_configure_sources_list.py | 5 +++++ tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml | 6 ++++++ 2 files changed, 11 insertions(+) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py index 129d2264..cf84e056 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py @@ -10,6 +10,11 @@ class TestAptconfigureSourcesList(base.CloudTestCase): def test_sources_list(self): """Test sources.list includes sources.""" out = self.get_data_file('sources.list') + + # Verify we have 6 entires + self.assertEqual(6, len(out.rstrip().split('\n'))) + + # Verify the keys generated the list correctly self.assertRegex(out, r'deb http:\/\/archive.ubuntu.com\/ubuntu ' '[a-z].* main restricted') self.assertRegex(out, r'deb-src http:\/\/archive.ubuntu.com\/ubuntu ' diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml index 143cb080..87e470c1 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml @@ -7,6 +7,12 @@ required_features: cloud_config: | #cloud-config apt: + primary: + - arches: [default] + uri: http://archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu sources_list: | deb $MIRROR $RELEASE main restricted deb-src $MIRROR $RELEASE main restricted -- cgit v1.2.3 From 5cc0b19b851a42f6a5edb0cc9d49dd76891b1bcb Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 9 Jan 2018 09:06:33 -0800 Subject: tests: remove zesty as supported OS to test Zesty goes EOL as of January 13, 2017. This removes it as a valid OS for testing. --- tests/cloud_tests/releases.yaml | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 0a9fa602..d8bc170f 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -158,22 +158,6 @@ releases: alias: artful setup_overrides: null override_templates: false - zesty: - # EOL: Jan 2018 - default: - enabled: true - release: zesty - version: 17.04 - os: ubuntu - feature_groups: - - base - - debian_base - - ubuntu_specific - lxd: - sstreams_server: https://cloud-images.ubuntu.com/daily - alias: zesty - setup_overrides: null - override_templates: false xenial: # EOL: Apr 2021 default: -- cgit v1.2.3 From c02a4d4c88cc2c6ec9f03ddf86703f5b67e04348 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 23 Jan 2018 15:05:35 -0700 Subject: tests: when querying ntp server, do not do dns resolution. Tests run on EC2 would successfully resolve the ipv4 dns address and that caused false positives on failure reports. Basically, dns lookup of 172.16.15.14 would return ip-172-16-15-14.us-east-2.compute.internal. which then shows up in the ntpq output unless you provide -n. --- tests/cloud_tests/testcases/modules/ntp_pools.yaml | 2 +- tests/cloud_tests/testcases/modules/ntp_servers.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml index 3a93faa2..d490b228 100644 --- a/tests/cloud_tests/testcases/modules/ntp_pools.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml @@ -26,6 +26,6 @@ collect_scripts: grep '^pool' /etc/ntp.conf ntpq_servers: | #!/bin/sh - ntpq -p -w + ntpq -p -w -n # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml index d59d45a8..6b13b70e 100644 --- a/tests/cloud_tests/testcases/modules/ntp_servers.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml @@ -22,6 +22,6 @@ collect_scripts: grep '^server' /etc/ntp.conf ntpq_servers: | #!/bin/sh - ntpq -p -w + ntpq -p -w -n # vi: ts=4 expandtab -- cgit v1.2.3 From 32a6a1764e902c31dd3af9b674cea14cd6501187 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 19 Jan 2018 09:43:55 -0500 Subject: tests: Fix EC2 Platform to return console output as bytes. The EC2 test platform uses boto, and boto decodes console output with decode('utf-8', 'replace). It is known that Ubuntu consoles contain non-utf8 characters, making this call lossy. The change here is to patch the boto session to include a OutputBytes entry in the console_output response, and then to utilize that in console_log. More information on problem and solution at: https://github.com/boto/botocore/issues/1351 --- tests/cloud_tests/platforms/ec2/instance.py | 10 +++++++-- tests/cloud_tests/platforms/ec2/platform.py | 33 ++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index 4ba737ab..ab6037b1 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -46,9 +46,15 @@ class EC2Instance(Instance): may return empty string. """ try: - return self.instance.console_output()['Output'].encode() + # OutputBytes comes from platform._decode_console_output_as_bytes + response = self.instance.console_output() + return response['OutputBytes'] except KeyError: - return b'' + if 'Output' in response: + msg = ("'OutputBytes' did not exist in console_output() but " + "'Output' did: %s..." % response['Output'][0:128]) + raise util.PlatformError('console_log', msg) + return ('No Console Output [%s]' % self.instance).encode() def destroy(self): """Clean up instance.""" diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index fdb17ba0..f188c27b 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -6,6 +6,8 @@ import os import boto3 import botocore +from botocore import session, handlers +import base64 from ..platforms import Platform from .image import EC2Image @@ -28,9 +30,10 @@ class EC2Platform(Platform): self.instance_type = config['instance-type'] try: - self.ec2_client = boto3.client('ec2') - self.ec2_resource = boto3.resource('ec2') - self.ec2_region = boto3.Session().region_name + b3session = get_session() + self.ec2_client = b3session.client('ec2') + self.ec2_resource = b3session.resource('ec2') + self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) except botocore.exceptions.NoRegionError: raise RuntimeError( @@ -228,4 +231,28 @@ class EC2Platform(Platform): return self.tag + +def _decode_console_output_as_bytes(parsed, **kwargs): + """Provide console output as bytes in OutputBytes. + + For this to be useful, the session has to have had the + decode_console_output handler unregistered already. + + https://github.com/boto/botocore/issues/1351 .""" + if 'Output' not in parsed: + return + orig = parsed['Output'] + handlers.decode_console_output(parsed, **kwargs) + parsed['OutputBytes'] = base64.b64decode(orig) + + +def get_session(): + mysess = session.get_session() + mysess.unregister('after-call.ec2.GetConsoleOutput', + handlers.decode_console_output) + mysess.register('after-call.ec2.GetConsoleOutput', + _decode_console_output_as_bytes) + return boto3.Session(botocore_session=mysess) + + # vi: ts=4 expandtab -- cgit v1.2.3 From bc84f5023f795c261e32cf0690b2d29e12cfaedd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jan 2018 12:26:34 -0700 Subject: tests: Collect script output as binary, collect systemd journal, fix lxd. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds collection a gzip compressed systemd journal on systemd systems. The file can later be reviewed with:   zcat system.journal.gz > system.journal   journalctl --file=system.journal [-o short-monotonic ..] To support this:   * modify test harness infrastructure to not assume content is utf-8.   * fix lxd platform to support make '_execute' return bytes rather     than a string. https://github.com/lxc/pylxd/issues/268 Also switched the base collectors to use /bin/sh as others already did. --- tests/cloud_tests/collect.py | 7 +++++ tests/cloud_tests/platforms/lxd/instance.py | 43 +++++++++++++---------------- tests/cloud_tests/testcases.yaml | 27 ++++++++++++++---- tests/cloud_tests/testcases/base.py | 6 ++-- tests/cloud_tests/verify.py | 2 +- 5 files changed, 52 insertions(+), 33 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 33acbb1e..5ea88e50 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -24,6 +24,13 @@ def collect_script(instance, base_dir, script, script_name): (out, err, exit) = instance.run_script( script.encode(), rcs=False, description='collect: {}'.format(script_name)) + if err: + LOG.debug("collect script %s had stderr: %s", script_name, err) + if not isinstance(out, bytes): + raise util.PlatformError( + "Collection of '%s' returned type %s, expected bytes: %s" % + (script_name, type(out), out)) + c_util.write_file(os.path.join(base_dir, script_name), out) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 0d697c05..d2d2a1fd 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -6,6 +6,8 @@ import os import shutil from tempfile import mkdtemp +from cloudinit.util import subp, ProcessExecutionError + from ..instances import Instance @@ -29,6 +31,7 @@ class LXDInstance(Instance): platform, name, properties, config, features) self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) self._setup_console_log() + self.name = name @property def pylxd_container(self): @@ -55,33 +58,25 @@ class LXDInstance(Instance): if env is None: env = {} - if stdin is not None: - # pylxd does not support input to execute. - # https://github.com/lxc/pylxd/issues/244 - # - # The solution here is write a tmp file in the container - # and then execute a shell that sets it standard in to - # be from that file, removes it, and calls the comand. - tmpf = self.tmpfile() - self.write_data(tmpf, stdin) - ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"' - command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] + - list(command)) + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] # ensure instance is running and execute the command self.start() - # execute returns a ContainerExecuteResult, named tuple - # (exit_code, stdout, stderr) - res = self.pylxd_container.execute(command, environment=env) - - # get out, exit and err from pylxd return - if not hasattr(res, 'exit_code'): - # pylxd 2.1.3 and earlier only return out and err, no exit - raise RuntimeError( - "No 'exit_code' in pylxd.container.execute return.\n" - "pylxd > 2.2 is required.") - - return res.stdout, res.stderr, res.exit_code + + # Use cmdline client due to https://github.com/lxc/pylxd/issues/268 + exit_code = 0 + try: + stdout, stderr = subp( + ['lxc', 'exec', self.name, '--'] + env_args + list(command), + data=stdin, decode=False) + except ProcessExecutionError as e: + exit_code = e.exit_code + stdout = e.stdout + stderr = e.stderr + + return stdout, stderr, exit_code def read_data(self, remote_path, decode=False): """Read data from instance filesystem. diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index 7183e017..8e0fb62f 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -7,22 +7,37 @@ base_test_data: #cloud-config collect_scripts: cloud-init.log: | - #!/bin/bash + #!/bin/sh cat /var/log/cloud-init.log cloud-init-output.log: | - #!/bin/bash + #!/bin/sh cat /var/log/cloud-init-output.log instance-id: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/.instance-id result.json: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/result.json status.json: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/status.json cloud-init-version: | - #!/bin/bash + #!/bin/sh dpkg-query -W -f='${Version}' cloud-init + system.journal.gz: | + #!/bin/sh + [ -d /run/systemd ] || { echo "not systemd."; exit 0; } + fail() { echo "ERROR:" "$@" 1>&2; exit 1; } + journal="" + for d in /run/log/journal /var/log/journal; do + for f in $d/*/system.journal; do + [ -f "$f" ] || continue + [ -z "$journal" ] || + fail "multiple journal found: $f $journal." + journal="$f" + done + done + [ -f "$journal" ] || fail "no journal file found." + gzip --to-stdout "$journal" # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 1c5b5405..20e95955 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -30,12 +30,14 @@ class CloudTestCase(unittest.TestCase): raise AssertionError('Key "{}" not in cloud config'.format(name)) return self.cloud_config[name] - def get_data_file(self, name): + def get_data_file(self, name, decode=True): """Get data file failing test if it is not present.""" if name not in self.data: raise AssertionError('File "{}" missing from collect data' .format(name)) - return self.data[name] + if not decode: + return self.data[name] + return self.data[name].decode('utf-8') def get_instance_id(self): """Get recorded instance id.""" diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index fc1efcfc..2a9fd520 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -29,7 +29,7 @@ def verify_data(base_dir, tests): data = {} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): - with open(os.path.join(test_dir, script_name), 'r') as fp: + with open(os.path.join(test_dir, script_name), 'rb') as fp: data[script_name] = fp.read() # get test suite and launch tests -- cgit v1.2.3