tests: consolidate platforms into specific dirs

This groups up each test platform into its own directory rather
than having files spread between four different directories for
one platform. Platforms tend to be worked on one at a time and
so having the platforms together makes more sense than apart.

4 files changed, 511 insertions, 0 deletions

diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py
new file mode 100644
index 00000000..b5de1f52
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/image.py
@@ -0,0 +1,193 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""LXD Image Base Class."""
+
+import os
+import shutil
+import tempfile
+
+from ..images import Image
+from .snapshot import LXDSnapshot
+from cloudinit import util as c_util
+from tests.cloud_tests import util
+
+
+class LXDImage(Image):
+    """LXD backed image."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, config, pylxd_image):
+        """Set up image.
+
+        @param platform: platform object
+        @param config: image configuration
+        """
+        self.modified = False
+        self._img_instance = None
+        self._pylxd_image = None
+        self.pylxd_image = pylxd_image
+        super(LXDImage, self).__init__(platform, config)
+
+    @property
+    def pylxd_image(self):
+        """Property function."""
+        if self._pylxd_image:
+            self._pylxd_image.sync()
+        return self._pylxd_image
+
+    @pylxd_image.setter
+    def pylxd_image(self, pylxd_image):
+        if self._img_instance:
+            self._instance.destroy()
+            self._img_instance = None
+        if (self._pylxd_image and
+                (self._pylxd_image is not pylxd_image) and
+                (not self.config.get('cache_base_image') or self.modified)):
+            self._pylxd_image.delete(wait=True)
+        self.modified = False
+        self._pylxd_image = pylxd_image
+
+    @property
+    def _instance(self):
+        """Internal use only, returns a instance
+
+        This starts an lxc instance from the image, so it is "dirty".
+        Better would be some way to modify this "at rest".
+        lxc-pstart would be an option."""
+        if not self._img_instance:
+            self._img_instance = self.platform.launch_container(
+                self.properties, self.config, self.features,
+                use_desc='image-modification', image_desc=str(self),
+                image=self.pylxd_image.fingerprint)
+            self._img_instance.start()
+        return self._img_instance
+
+    @property
+    def properties(self):
+        """{} containing: 'arch', 'os', 'version', 'release'."""
+        properties = self.pylxd_image.properties
+        return {
+            'arch': properties.get('architecture'),
+            'os': properties.get('os'),
+            'version': properties.get('version'),
+            'release': properties.get('release'),
+        }
+
+    def export_image(self, output_dir):
+        """Export image from lxd image store to (split) tarball on disk.
+
+        @param output_dir: dir to store tarballs in
+        @return_value: tuple of path to metadata tarball and rootfs tarball
+        """
+        # pylxd's image export feature doesn't do split exports, so use cmdline
+        c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint,
+                     output_dir], capture=True)
+        tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')]
+        metadata = os.path.join(
+            output_dir, next(p for p in tarballs if p.startswith('meta-')))
+        rootfs = os.path.join(
+            output_dir, next(p for p in tarballs if not p.startswith('meta-')))
+        return (metadata, rootfs)
+
+    def import_image(self, metadata, rootfs):
+        """Import image to lxd image store from (split) tarball on disk.
+
+        Note, this will replace and delete the current pylxd_image
+
+        @param metadata: metadata tarball
+        @param rootfs: rootfs tarball
+        @return_value: imported image fingerprint
+        """
+        alias = util.gen_instance_name(
+            image_desc=str(self), use_desc='update-metadata')
+        c_util.subp(['lxc', 'image', 'import', metadata, rootfs,
+                     '--alias', alias], capture=True)
+        self.pylxd_image = self.platform.query_image_by_alias(alias)
+        return self.pylxd_image.fingerprint
+
+    def update_templates(self, template_config, template_data):
+        """Update the image's template configuration.
+
+        Note, this will replace and delete the current pylxd_image
+
+        @param template_config: config overrides for template metadata
+        @param template_data: template data to place into templates/
+        """
+        # set up tmp files
+        export_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
+        extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
+        new_metadata = os.path.join(export_dir, 'new-meta.tar.xz')
+        metadata_yaml = os.path.join(extract_dir, 'metadata.yaml')
+        template_dir = os.path.join(extract_dir, 'templates')
+
+        try:
+            # extract old data
+            (metadata, rootfs) = self.export_image(export_dir)
+            shutil.unpack_archive(metadata, extract_dir)
+
+            # update metadata
+            metadata = c_util.read_conf(metadata_yaml)
+            templates = metadata.get('templates', {})
+            templates.update(template_config)
+            metadata['templates'] = templates
+            util.yaml_dump(metadata, metadata_yaml)
+
+            # write out template files
+            for name, content in template_data.items():
+                path = os.path.join(template_dir, name)
+                c_util.write_file(path, content)
+
+            # store new data, mark new image as modified
+            util.flat_tar(new_metadata, extract_dir)
+            self.import_image(new_metadata, rootfs)
+            self.modified = True
+
+        finally:
+            # remove tmpfiles
+            shutil.rmtree(export_dir)
+            shutil.rmtree(extract_dir)
+
+    def _execute(self, *args, **kwargs):
+        """Execute command in image, modifying image."""
+        return self._instance._execute(*args, **kwargs)
+
+    def push_file(self, local_path, remote_path):
+        """Copy file at 'local_path' to instance at 'remote_path'."""
+        return self._instance.push_file(local_path, remote_path)
+
+    def run_script(self, *args, **kwargs):
+        """Run script in image, modifying image.
+
+        @return_value: script output
+        """
+        return self._instance.run_script(*args, **kwargs)
+
+    def snapshot(self):
+        """Create snapshot of image, block until done."""
+        # get empty user data to pass in to instance
+        # if overrides for user data provided, use them
+        empty_userdata = util.update_user_data(
+            {}, self.config.get('user_data_overrides', {}))
+        conf = {'user.user-data': empty_userdata}
+        # clone current instance
+        instance = self.platform.launch_container(
+            self.properties, self.config, self.features,
+            container=self._instance.name, image_desc=str(self),
+            use_desc='snapshot', container_config=conf)
+        # wait for cloud-init before boot_clean_script is run to ensure
+        # /var/lib/cloud is removed cleanly
+        instance.start(wait=True, wait_for_cloud_init=True)
+        if self.config.get('boot_clean_script'):
+            instance.run_script(self.config.get('boot_clean_script'))
+        # freeze current instance and return snapshot
+        instance.freeze()
+        return LXDSnapshot(self.platform, self.properties, self.config,
+                           self.features, instance)
+
+    def destroy(self):
+        """Clean up data associated with image."""
+        self.pylxd_image = None
+        super(LXDImage, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
new file mode 100644
index 00000000..0d697c05
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -0,0 +1,157 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD instance."""
+
+import os
+import shutil
+from tempfile import mkdtemp
+
+from ..instances import Instance
+
+
+class LXDInstance(Instance):
+    """LXD container backed instance."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, name, properties, config, features,
+                 pylxd_container):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: hostname of instance
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self._pylxd_container = pylxd_container
+        super(LXDInstance, self).__init__(
+            platform, name, properties, config, features)
+        self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name))
+        self._setup_console_log()
+
+    @property
+    def pylxd_container(self):
+        """Property function."""
+        self._pylxd_container.sync()
+        return self._pylxd_container
+
+    def _setup_console_log(self):
+        logf = os.path.join(self.tmpd, "console.log")
+
+        # doing this ensures we can read it. Otherwise it ends up root:root.
+        with open(logf, "w") as fp:
+            fp.write("# %s\n" % self.name)
+
+        cfg = "lxc.console.logfile=%s" % logf
+        orig = self._pylxd_container.config.get('raw.lxc', "")
+        if orig:
+            orig += "\n"
+        self._pylxd_container.config['raw.lxc'] = orig + cfg
+        self._pylxd_container.save()
+        self._console_log_file = logf
+
+    def _execute(self, command, stdin=None, env=None):
+        if env is None:
+            env = {}
+
+        if stdin is not None:
+            # pylxd does not support input to execute.
+            # https://github.com/lxc/pylxd/issues/244
+            #
+            # The solution here is write a tmp file in the container
+            # and then execute a shell that sets it standard in to
+            # be from that file, removes it, and calls the comand.
+            tmpf = self.tmpfile()
+            self.write_data(tmpf, stdin)
+            ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"'
+            command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] +
+                       list(command))
+
+        # ensure instance is running and execute the command
+        self.start()
+        # execute returns a ContainerExecuteResult, named tuple
+        # (exit_code, stdout, stderr)
+        res = self.pylxd_container.execute(command, environment=env)
+
+        # get out, exit and err from pylxd return
+        if not hasattr(res, 'exit_code'):
+            # pylxd 2.1.3 and earlier only return out and err, no exit
+            raise RuntimeError(
+                "No 'exit_code' in pylxd.container.execute return.\n"
+                "pylxd > 2.2 is required.")
+
+        return res.stdout, res.stderr, res.exit_code
+
+    def read_data(self, remote_path, decode=False):
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: decode data before returning.
+        @return_value: content of remote_path as bytes if 'decode' is False,
+                       and as string if 'decode' is True.
+        """
+        data = self.pylxd_container.files.get(remote_path)
+        return data.decode() if decode else data
+
+    def write_data(self, remote_path, data):
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write in bytes
+        """
+        self.pylxd_container.files.put(remote_path, data)
+
+    def console_log(self):
+        """Console log.
+
+        @return_value: bytes of this instance’s console
+        """
+        if not os.path.exists(self._console_log_file):
+            raise NotImplementedError(
+                "Console log '%s' does not exist. If this is a remote "
+                "lxc, then this is really NotImplementedError.  If it is "
+                "A local lxc, then this is a RuntimeError."
+                "https://github.com/lxc/lxd/issues/1129")
+        with open(self._console_log_file, "rb") as fp:
+            return fp.read()
+
+    def reboot(self, wait=True):
+        """Reboot instance."""
+        self.shutdown(wait=wait)
+        self.start(wait=wait)
+
+    def shutdown(self, wait=True):
+        """Shutdown instance."""
+        if self.pylxd_container.status != 'Stopped':
+            self.pylxd_container.stop(wait=wait)
+
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
+        if self.pylxd_container.status != 'Running':
+            self.pylxd_container.start(wait=wait)
+            if wait:
+                self._wait_for_system(wait_for_cloud_init)
+
+    def freeze(self):
+        """Freeze instance."""
+        if self.pylxd_container.status != 'Frozen':
+            self.pylxd_container.freeze(wait=True)
+
+    def unfreeze(self):
+        """Unfreeze instance."""
+        if self.pylxd_container.status == 'Frozen':
+            self.pylxd_container.unfreeze(wait=True)
+
+    def destroy(self):
+        """Clean up instance."""
+        self.unfreeze()
+        self.shutdown()
+        self.pylxd_container.delete(wait=True)
+        if self.platform.container_exists(self.name):
+            raise OSError('container {} was not properly removed'
+                          .format(self.name))
+        shutil.rmtree(self.tmpd)
+        super(LXDInstance, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py
new file mode 100644
index 00000000..6a016929
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/platform.py
@@ -0,0 +1,108 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD platform."""
+
+from pylxd import (Client, exceptions)
+
+from ..platforms import Platform
+from .image import LXDImage
+from .instance import LXDInstance
+from tests.cloud_tests import util
+
+DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443"
+
+
+class LXDPlatform(Platform):
+    """LXD test platform."""
+
+    platform_name = 'lxd'
+
+    def __init__(self, config):
+        """Set up platform."""
+        super(LXDPlatform, self).__init__(config)
+        # TODO: allow configuration of remote lxd host via env variables
+        # set up lxd connection
+        self.client = Client()
+
+    def get_image(self, img_conf):
+        """Get image using specified image configuration.
+
+        @param img_conf: configuration for image
+        @return_value: cloud_tests.images instance
+        """
+        pylxd_image = self.client.images.create_from_simplestreams(
+            img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER),
+            img_conf['alias'])
+        image = LXDImage(self, img_conf, pylxd_image)
+        if img_conf.get('override_templates', False):
+            image.update_templates(self.config.get('template_overrides', {}),
+                                   self.config.get('template_files', {}))
+        return image
+
+    def launch_container(self, properties, config, features,
+                         image=None, container=None, ephemeral=False,
+                         container_config=None, block=True, image_desc=None,
+                         use_desc=None):
+        """Launch a container.
+
+        @param properties: image properties
+        @param config: image configuration
+        @param features: image features
+        @param image: image fingerprint to launch from
+        @param container: container to copy
+        @param ephemeral: delete image after first shutdown
+        @param container_config: config options for instance as dict
+        @param block: wait until container created
+        @param image_desc: description of image being launched
+        @param use_desc: description of container's use
+        @return_value: cloud_tests.instances instance
+        """
+        if not (image or container):
+            raise ValueError("either image or container must be specified")
+        container = self.client.containers.create({
+            'name': util.gen_instance_name(image_desc=image_desc,
+                                           use_desc=use_desc,
+                                           used_list=self.list_containers()),
+            'ephemeral': bool(ephemeral),
+            'config': (container_config
+                       if isinstance(container_config, dict) else {}),
+            'source': ({'type': 'image', 'fingerprint': image} if image else
+                       {'type': 'copy', 'source': container})
+        }, wait=block)
+        return LXDInstance(self, container.name, properties, config, features,
+                           container)
+
+    def container_exists(self, container_name):
+        """Check if container with name 'container_name' exists.
+
+        @return_value: True if exists else False
+        """
+        res = True
+        try:
+            self.client.containers.get(container_name)
+        except exceptions.LXDAPIException as e:
+            res = False
+            if e.response.status_code != 404:
+                raise
+        return res
+
+    def list_containers(self):
+        """List names of all containers.
+
+        @return_value: list of names
+        """
+        return [container.name for container in self.client.containers.all()]
+
+    def query_image_by_alias(self, alias):
+        """Get image by alias in local image store.
+
+        @param alias: alias of image
+        @return_value: pylxd image (not cloud_tests.images instance)
+        """
+        return self.client.images.get_by_alias(alias)
+
+    def destroy(self):
+        """Clean up platform data."""
+        super(LXDPlatform, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/snapshot.py b/tests/cloud_tests/platforms/lxd/snapshot.py
new file mode 100644
index 00000000..b524644f
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/snapshot.py
@@ -0,0 +1,53 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD snapshot."""
+
+from ..snapshots import Snapshot
+
+
+class LXDSnapshot(Snapshot):
+    """LXD image copy backed snapshot."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, properties, config, features,
+                 pylxd_frozen_instance):
+        """Set up snapshot.
+
+        @param platform: platform object
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self.pylxd_frozen_instance = pylxd_frozen_instance
+        super(LXDSnapshot, self).__init__(
+            platform, properties, config, features)
+
+    def launch(self, user_data, meta_data=None, block=True, start=True,
+               use_desc=None):
+        """Launch instance.
+
+        @param user_data: user-data for the instance
+        @param instance_id: instance-id for the instance
+        @param block: wait until instance is created
+        @param start: start instance and wait until fully started
+        @param use_desc: description of snapshot instance use
+        @return_value: an Instance
+        """
+        inst_config = {'user.user-data': user_data}
+        if meta_data:
+            inst_config['user.meta-data'] = meta_data
+        instance = self.platform.launch_container(
+            self.properties, self.config, self.features, block=block,
+            image_desc=str(self), container=self.pylxd_frozen_instance.name,
+            use_desc=use_desc, container_config=inst_config)
+        if start:
+            instance.start()
+        return instance
+
+    def destroy(self):
+        """Clean up snapshot data."""
+        self.pylxd_frozen_instance.destroy()
+        super(LXDSnapshot, self).destroy()
+
+# vi: ts=4 expandtab