4 files changed, 511 insertions, 0 deletions
diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py
new file mode 100644
index 00000000..b5de1f52
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/image.py
@@ -0,0 +1,193 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""LXD Image Base Class."""
+
+import os
+import shutil
+import tempfile
+
+from ..images import Image
+from .snapshot import LXDSnapshot
+from cloudinit import util as c_util
+from tests.cloud_tests import util
+
+
+class LXDImage(Image):
+    """LXD backed image."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, config, pylxd_image):
+        """Set up image.
+
+        @param platform: platform object
+        @param config: image configuration
+        """
+        self.modified = False
+        self._img_instance = None
+        self._pylxd_image = None
+        self.pylxd_image = pylxd_image
+        super(LXDImage, self).__init__(platform, config)
+
+    @property
+    def pylxd_image(self):
+        """Property function."""
+        if self._pylxd_image:
+            self._pylxd_image.sync()
+        return self._pylxd_image
+
+    @pylxd_image.setter
+    def pylxd_image(self, pylxd_image):
+        if self._img_instance:
+            self._instance.destroy()
+            self._img_instance = None
+        if (self._pylxd_image and
+                (self._pylxd_image is not pylxd_image) and
+                (not self.config.get('cache_base_image') or self.modified)):
+            self._pylxd_image.delete(wait=True)
+        self.modified = False
+        self._pylxd_image = pylxd_image
+
+    @property
+    def _instance(self):
+        """Internal use only, returns a instance
+
+        This starts an lxc instance from the image, so it is "dirty".
+        Better would be some way to modify this "at rest".
+        lxc-pstart would be an option."""
+        if not self._img_instance:
+            self._img_instance = self.platform.launch_container(
+                self.properties, self.config, self.features,
+                use_desc='image-modification', image_desc=str(self),
+                image=self.pylxd_image.fingerprint)
+            self._img_instance.start()
+        return self._img_instance
+
+    @property
+    def properties(self):
+        """{} containing: 'arch', 'os', 'version', 'release'."""
+        properties = self.pylxd_image.properties
+        return {
+            'arch': properties.get('architecture'),
+            'os': properties.get('os'),
+            'version': properties.get('version'),
+            'release': properties.get('release'),
+        }
+
+    def export_image(self, output_dir):
+        """Export image from lxd image store to (split) tarball on disk.
+
+        @param output_dir: dir to store tarballs in
+        @return_value: tuple of path to metadata tarball and rootfs tarball
+        """
+        # pylxd's image export feature doesn't do split exports, so use cmdline
+        c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint,
+                     output_dir], capture=True)
+        tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')]
+        metadata = os.path.join(
+            output_dir, next(p for p in tarballs if p.startswith('meta-')))
+        rootfs = os.path.join(
+            output_dir, next(p for p in tarballs if not p.startswith('meta-')))
+        return (metadata, rootfs)
+
+    def import_image(self, metadata, rootfs):
+        """Import image to lxd image store from (split) tarball on disk.
+
+        Note, this will replace and delete the current pylxd_image
+
+        @param metadata: metadata tarball
+        @param rootfs: rootfs tarball
+        @return_value: imported image fingerprint
+        """
+        alias = util.gen_instance_name(
+            image_desc=str(self), use_desc='update-metadata')
+        c_util.subp(['lxc', 'image', 'import', metadata, rootfs,
+                     '--alias', alias], capture=True)
+        self.pylxd_image = self.platform.query_image_by_alias(alias)
+        return self.pylxd_image.fingerprint
+
+    def update_templates(self, template_config, template_data):
+        """Update the image's template configuration.
+
+        Note, this will replace and delete the current pylxd_image
+
+        @param template_config: config overrides for template metadata
+        @param template_data: template data to place into templates/
+        """
+        # set up tmp files
+        export_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
+        extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
+        new_metadata = os.path.join(export_dir, 'new-meta.tar.xz')
+        metadata_yaml = os.path.join(extract_dir, 'metadata.yaml')
+        template_dir = os.path.join(extract_dir, 'templates')
+
+        try:
+            # extract old data
+            (metadata, rootfs) = self.export_image(export_dir)
+            shutil.unpack_archive(metadata, extract_dir)
+
+            # update metadata
+            metadata = c_util.read_conf(metadata_yaml)
+            templates = metadata.get('templates', {})
+            templates.update(template_config)
+            metadata['templates'] = templates
+            util.yaml_dump(metadata, metadata_yaml)
+
+            # write out template files
+            for name, content in template_data.items():
+                path = os.path.join(template_dir, name)
+                c_util.write_file(path, content)
+
+            # store new data, mark new image as modified
+            util.flat_tar(new_metadata, extract_dir)
+            self.import_image(new_metadata, rootfs)
+            self.modified = True
+
+        finally:
+            # remove tmpfiles
+            shutil.rmtree(export_dir)
+            shutil.rmtree(extract_dir)
+
+    def _execute(self, *args, **kwargs):
+        """Execute command in image, modifying image."""
+        return self._instance._execute(*args, **kwargs)
+
+    def push_file(self, local_path, remote_path):
+        """Copy file at 'local_path' to instance at 'remote_path'."""
+        return self._instance.push_file(local_path, remote_path)
+
+    def run_script(self, *args, **kwargs):
+        """Run script in image, modifying image.
+
+        @return_value: script output
+        """
+        return self._instance.run_script(*args, **kwargs)
+
+    def snapshot(self):
+        """Create snapshot of image, block until done."""
+        # get empty user data to pass in to instance
+        # if overrides for user data provided, use them
+        empty_userdata = util.update_user_data(
+            {}, self.config.get('user_data_overrides', {}))
+        conf = {'user.user-data': empty_userdata}
+        # clone current instance
+        instance = self.platform.launch_container(
+            self.properties, self.config, self.features,
+            container=self._instance.name, image_desc=str(self),
+            use_desc='snapshot', container_config=conf)
+        # wait for cloud-init before boot_clean_script is run to ensure
+        # /var/lib/cloud is removed cleanly
+        instance.start(wait=True, wait_for_cloud_init=True)
+        if self.config.get('boot_clean_script'):
+            instance.run_script(self.config.get('boot_clean_script'))
+        # freeze current instance and return snapshot
+        instance.freeze()
+        return LXDSnapshot(self.platform, self.properties, self.config,
+                           self.features, instance)
+
+    def destroy(self):
+        """Clean up data associated with image."""
+        self.pylxd_image = None
+        super(LXDImage, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
new file mode 100644
index 00000000..0d697c05
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -0,0 +1,157 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD instance."""
+
+import os
+import shutil
+from tempfile import mkdtemp
+
+from ..instances import Instance
+
+
+class LXDInstance(Instance):
+    """LXD container backed instance."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, name, properties, config, features,
+                 pylxd_container):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: hostname of instance
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self._pylxd_container = pylxd_container
+        super(LXDInstance, self).__init__(
+            platform, name, properties, config, features)
+        self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name))
+        self._setup_console_log()
+
+    @property
+    def pylxd_container(self):
+        """Property function."""
+        self._pylxd_container.sync()
+        return self._pylxd_container
+
+    def _setup_console_log(self):
+        logf = os.path.join(self.tmpd, "console.log")
+
+        # doing this ensures we can read it. Otherwise it ends up root:root.
+        with open(logf, "w") as fp:
+            fp.write("# %s\n" % self.name)
+
+        cfg = "lxc.console.logfile=%s" % logf
+        orig = self._pylxd_container.config.get('raw.lxc', "")
+        if orig:
+            orig += "\n"
+        self._pylxd_container.config['raw.lxc'] = orig + cfg
+        self._pylxd_container.save()
+        self._console_log_file = logf
+
+    def _execute(self, command, stdin=None, env=None):
+        if env is None:
+            env = {}
+
+        if stdin is not None:
+            # pylxd does not support input to execute.
+            # https://github.com/lxc/pylxd/issues/244
+            #
+            # The solution here is write a tmp file in the container
+            # and then execute a shell that sets it standard in to
+            # be from that file, removes it, and calls the comand.
+            tmpf = self.tmpfile()
+            self.write_data(tmpf, stdin)
+            ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"'
+            command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] +
+                       list(command))
+
+        # ensure instance is running and execute the command
+        self.start()
+        # execute returns a ContainerExecuteResult, named tuple
+        # (exit_code, stdout, stderr)
+        res = self.pylxd_container.execute(command, environment=env)
+
+        # get out, exit and err from pylxd return
+        if not hasattr(res, 'exit_code'):
+            # pylxd 2.1.3 and earlier only return out and err, no exit
+            raise RuntimeError(
+                "No 'exit_code' in pylxd.container.execute return.\n"
+                "pylxd > 2.2 is required.")
+
+        return res.stdout, res.stderr, res.exit_code
+
+    def read_data(self, remote_path, decode=False):
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: decode data before returning.
+        @return_value: content of remote_path as bytes if 'decode' is False,
+                       and as string if 'decode' is True.
+        """
+        data = self.pylxd_container.files.get(remote_path)
+        return data.decode() if decode else data
+
+    def write_data(self, remote_path, data):
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write in bytes
+        """
+        self.pylxd_container.files.put(remote_path, data)
+
+    def console_log(self):
+        """Console log.
+
+        @return_value: bytes of this instance’s console
+        """
+        if not os.path.exists(self._console_log_file):
+            raise NotImplementedError(
+                "Console log '%s' does not exist. If this is a remote "
+                "lxc, then this is really NotImplementedError.  If it is "
+                "A local lxc, then this is a RuntimeError."
+                "https://github.com/lxc/lxd/issues/1129")
+        with open(self._console_log_file, "rb") as fp:
+            return fp.read()
+
+    def reboot(self, wait=True):
+        """Reboot instance."""
+        self.shutdown(wait=wait)
+        self.start(wait=wait)
+
+    def shutdown(self, wait=True):
+        """Shutdown instance."""
+        if self.pylxd_container.status != 'Stopped':
+            self.pylxd_container.stop(wait=wait)
+
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
+        if self.pylxd_container.status != 'Running':
+            self.pylxd_container.start(wait=wait)
+            if wait:
+                self._wait_for_system(wait_for_cloud_init)
+
+    def freeze(self):
+        """Freeze instance."""
+        if self.pylxd_container.status != 'Frozen':
+            self.pylxd_container.freeze(wait=True)
+
+    def unfreeze(self):
+        """Unfreeze instance."""
+        if self.pylxd_container.status == 'Frozen':
+            self.pylxd_container.unfreeze(wait=True)
+
+    def destroy(self):
+        """Clean up instance."""
+        self.unfreeze()
+        self.shutdown()
+        self.pylxd_container.delete(wait=True)
+        if self.platform.container_exists(self.name):
+            raise OSError('container {} was not properly removed'
+                          .format(self.name))
+        shutil.rmtree(self.tmpd)
+        super(LXDInstance, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py
new file mode 100644
index 00000000..6a016929
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/platform.py
@@ -0,0 +1,108 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD platform."""
+
+from pylxd import (Client, exceptions)
+
+from ..platforms import Platform
+from .image import LXDImage
+from .instance import LXDInstance
+from tests.cloud_tests import util
+
+DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443"
+
+
+class LXDPlatform(Platform):
+    """LXD test platform."""
+
+    platform_name = 'lxd'
+
+    def __init__(self, config):
+        """Set up platform."""
+        super(LXDPlatform, self).__init__(config)
+        # TODO: allow configuration of remote lxd host via env variables
+        # set up lxd connection
+        self.client = Client()
+
+    def get_image(self, img_conf):
+        """Get image using specified image configuration.
+
+        @param img_conf: configuration for image
+        @return_value: cloud_tests.images instance
+        """
+        pylxd_image = self.client.images.create_from_simplestreams(
+            img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER),
+            img_conf['alias'])
+        image = LXDImage(self, img_conf, pylxd_image)
+        if img_conf.get('override_templates', False):
+            image.update_templates(self.config.get('template_overrides', {}),
+                                   self.config.get('template_files', {}))
+        return image
+
+    def launch_container(self, properties, config, features,
+                         image=None, container=None, ephemeral=False,
+                         container_config=None, block=True, image_desc=None,
+                         use_desc=None):
+        """Launch a container.
+
+        @param properties: image properties
+        @param config: image configuration
+        @param features: image features
+        @param image: image fingerprint to launch from
+        @param container: container to copy
+        @param ephemeral: delete image after first shutdown
+        @param container_config: config options for instance as dict
+        @param block: wait until container created
+        @param image_desc: description of image being launched
+        @param use_desc: description of container's use
+        @return_value: cloud_tests.instances instance
+        """
+        if not (image or container):
+            raise ValueError("either image or container must be specified")
+        container = self.client.containers.create({
+            'name': util.gen_instance_name(image_desc=image_desc,
+                                           use_desc=use_desc,
+                                           used_list=self.list_containers()),
+            'ephemeral': bool(ephemeral),
+            'config': (container_config
+                       if isinstance(container_config, dict) else {}),
+            'source': ({'type': 'image', 'fingerprint': image} if image else
+                       {'type': 'copy', 'source': container})
+        }, wait=block)
+        return LXDInstance(self, container.name, properties, config, features,
+                           container)
+
+    def container_exists(self, container_name):
+        """Check if container with name 'container_name' exists.
+
+        @return_value: True if exists else False
+        """
+        res = True
+        try:
+            self.client.containers.get(container_name)
+        except exceptions.LXDAPIException as e:
+            res = False
+            if e.response.status_code != 404:
+                raise
+        return res
+
+    def list_containers(self):
+        """List names of all containers.
+
+        @return_value: list of names
+        """
+        return [container.name for container in self.client.containers.all()]
+
+    def query_image_by_alias(self, alias):
+        """Get image by alias in local image store.
+
+        @param alias: alias of image
+        @return_value: pylxd image (not cloud_tests.images instance)
+        """
+        return self.client.images.get_by_alias(alias)
+
+    def destroy(self):
+        """Clean up platform data."""
+        super(LXDPlatform, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/lxd/snapshot.py b/tests/cloud_tests/platforms/lxd/snapshot.py
new file mode 100644
index 00000000..b524644f
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/snapshot.py
@@ -0,0 +1,53 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base LXD snapshot."""
+
+from ..snapshots import Snapshot
+
+
+class LXDSnapshot(Snapshot):
+    """LXD image copy backed snapshot."""
+
+    platform_name = "lxd"
+
+    def __init__(self, platform, properties, config, features,
+                 pylxd_frozen_instance):
+        """Set up snapshot.
+
+        @param platform: platform object
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self.pylxd_frozen_instance = pylxd_frozen_instance
+        super(LXDSnapshot, self).__init__(
+            platform, properties, config, features)
+
+    def launch(self, user_data, meta_data=None, block=True, start=True,
+               use_desc=None):
+        """Launch instance.
+
+        @param user_data: user-data for the instance
+        @param instance_id: instance-id for the instance
+        @param block: wait until instance is created
+        @param start: start instance and wait until fully started
+        @param use_desc: description of snapshot instance use
+        @return_value: an Instance
+        """
+        inst_config = {'user.user-data': user_data}
+        if meta_data:
+            inst_config['user.meta-data'] = meta_data
+        instance = self.platform.launch_container(
+            self.properties, self.config, self.features, block=block,
+            image_desc=str(self), container=self.pylxd_frozen_instance.name,
+            use_desc=use_desc, container_config=inst_config)
+        if start:
+            instance.start()
+        return instance
+
+    def destroy(self):
+        """Clean up snapshot data."""
+        self.pylxd_frozen_instance.destroy()
+        super(LXDSnapshot, self).destroy()
+
+# vi: ts=4 expandtab