tests: consolidate platforms into specific dirs

This groups up each test platform into its own directory rather
than having files spread between four different directories for
one platform. Platforms tend to be worked on one at a time and
so having the platforms together makes more sense than apart.

4 files changed, 436 insertions, 0 deletions

diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py
new file mode 100644
index 00000000..09ff2a3b
--- /dev/null
+++ b/tests/cloud_tests/platforms/nocloudkvm/image.py
@@ -0,0 +1,89 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""NoCloud KVM Image Base Class."""
+
+from cloudinit import util as c_util
+
+import os
+import shutil
+import tempfile
+
+from ..images import Image
+from .snapshot import NoCloudKVMSnapshot
+
+
+class NoCloudKVMImage(Image):
+    """NoCloud KVM backed image."""
+
+    platform_name = "nocloud-kvm"
+
+    def __init__(self, platform, config, orig_img_path):
+        """Set up image.
+
+        @param platform: platform object
+        @param config: image configuration
+        @param img_path: path to the image
+        """
+        self.modified = False
+        self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage')
+        self._orig_img_path = orig_img_path
+        self._img_path = os.path.join(self._workd,
+                                      os.path.basename(self._orig_img_path))
+
+        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
+                    '-b', orig_img_path, self._img_path])
+
+        super(NoCloudKVMImage, self).__init__(platform, config)
+
+    @property
+    def properties(self):
+        """Dictionary containing: 'arch', 'os', 'version', 'release'."""
+        return {
+            'arch': self.config['arch'],
+            'os': self.config['family'],
+            'release': self.config['release'],
+            'version': self.config['version'],
+        }
+
+    def _execute(self, command, stdin=None, env=None):
+        """Execute command in image, modifying image."""
+        return self.mount_image_callback(command, stdin=stdin, env=env)
+
+    def mount_image_callback(self, command, stdin=None, env=None):
+        """Run mount-image-callback."""
+
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
+
+        mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts',
+                      '--system-resolvconf', self._img_path,
+                      '--', 'chroot', '_MOUNTPOINT_']
+        try:
+            out, err = c_util.subp(mic_chroot + env_args + list(command),
+                                   data=stdin, decode=False)
+            return (out, err, 0)
+        except c_util.ProcessExecutionError as e:
+            return (e.stdout, e.stderr, e.exit_code)
+
+    def snapshot(self):
+        """Create snapshot of image, block until done."""
+        if not self._img_path:
+            raise RuntimeError()
+
+        return NoCloudKVMSnapshot(self.platform, self.properties, self.config,
+                                  self.features, self._img_path)
+
+    def destroy(self):
+        """Unset path to signal image is no longer used.
+
+        The removal of the images and all other items is handled by the
+        framework. In some cases we want to keep the images, so let the
+        framework decide whether to keep or destroy everything.
+        """
+        self._img_path = None
+        shutil.rmtree(self._workd)
+
+        super(NoCloudKVMImage, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py
new file mode 100644
index 00000000..a87d76a6
--- /dev/null
+++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py
@@ -0,0 +1,179 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base NoCloud KVM instance."""
+
+import os
+import paramiko
+import socket
+import subprocess
+import time
+
+from ..instances import Instance
+from cloudinit import util as c_util
+from tests.cloud_tests import util
+
+# This domain contains reverse lookups for hostnames that are used.
+# The primary reason is so sudo will return quickly when it attempts
+# to look up the hostname.  i9n is just short for 'integration'.
+# see also bug 1730744 for why we had to do this.
+CI_DOMAIN = "i9n.cloud-init.io"
+
+
+class NoCloudKVMInstance(Instance):
+    """NoCloud KVM backed instance."""
+
+    platform_name = "nocloud-kvm"
+    _ssh_client = None
+
+    def __init__(self, platform, name, image_path, properties, config,
+                 features, user_data, meta_data):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: image path
+        @param image_path: path to disk image to boot.
+        @param properties: dictionary of properties
+        @param config: dictionary of configuration values
+        @param features: dictionary of supported feature flags
+        """
+        self.user_data = user_data
+        self.meta_data = meta_data
+        self.ssh_key_file = os.path.join(platform.config['data_dir'],
+                                         platform.config['private_key'])
+        self.ssh_port = None
+        self.pid = None
+        self.pid_file = None
+        self.console_file = None
+        self.disk = image_path
+
+        super(NoCloudKVMInstance, self).__init__(
+            platform, name, properties, config, features)
+
+    def destroy(self):
+        """Clean up instance."""
+        if self.pid:
+            try:
+                c_util.subp(['kill', '-9', self.pid])
+            except util.ProcessExectuionError:
+                pass
+
+        if self.pid_file:
+            os.remove(self.pid_file)
+
+        self.pid = None
+        if self._ssh_client:
+            self._ssh_client.close()
+            self._ssh_client = None
+
+        super(NoCloudKVMInstance, self).destroy()
+
+    def _execute(self, command, stdin=None, env=None):
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
+
+        return self.ssh(['sudo'] + env_args + list(command), stdin=stdin)
+
+    def generate_seed(self, tmpdir):
+        """Generate nocloud seed from user-data"""
+        seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name)
+        user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name)
+
+        with open(user_data_file, "w") as ud_file:
+            ud_file.write(self.user_data)
+
+        c_util.subp(['cloud-localds', seed_file, user_data_file])
+
+        return seed_file
+
+    def get_free_port(self):
+        """Get a free port assigned by the kernel."""
+        s = socket.socket()
+        s.bind(('', 0))
+        num = s.getsockname()[1]
+        s.close()
+        return num
+
+    def ssh(self, command, stdin=None):
+        """Run a command via SSH."""
+        client = self._ssh_connect()
+
+        cmd = util.shell_pack(command)
+        try:
+            fp_in, fp_out, fp_err = client.exec_command(cmd)
+            channel = fp_in.channel
+            if stdin is not None:
+                fp_in.write(stdin)
+                fp_in.close()
+
+            channel.shutdown_write()
+            rc = channel.recv_exit_status()
+            return (fp_out.read(), fp_err.read(), rc)
+        except paramiko.SSHException as e:
+            raise util.InTargetExecuteError(
+                b'', b'', -1, command, self.name, reason=e)
+
+    def _ssh_connect(self, hostname='localhost', username='ubuntu',
+                     banner_timeout=120, retry_attempts=30):
+        """Connect via SSH."""
+        if self._ssh_client:
+            return self._ssh_client
+
+        private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file)
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        while retry_attempts:
+            try:
+                client.connect(hostname=hostname, username=username,
+                               port=self.ssh_port, pkey=private_key,
+                               banner_timeout=banner_timeout)
+                self._ssh_client = client
+                return client
+            except (paramiko.SSHException, TypeError):
+                time.sleep(1)
+                retry_attempts = retry_attempts - 1
+
+        error_desc = 'Failed command to: %s@%s:%s' % (username, hostname,
+                                                      self.ssh_port)
+        raise util.InTargetExecuteError('', '', -1, 'ssh connect',
+                                        self.name, error_desc)
+
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
+        tmpdir = self.platform.config['data_dir']
+        seed = self.generate_seed(tmpdir)
+        self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name)
+        self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name)
+        self.ssh_port = self.get_free_port()
+
+        cmd = ['./tools/xkvm',
+               '--disk', '%s,cache=unsafe' % self.disk,
+               '--disk', '%s,cache=unsafe' % seed,
+               '--netdev', ','.join(['user',
+                                     'hostfwd=tcp::%s-:22' % self.ssh_port,
+                                     'dnssearch=%s' % CI_DOMAIN]),
+               '--', '-pidfile', self.pid_file, '-vnc', 'none',
+               '-m', '2G', '-smp', '2', '-nographic',
+               '-serial', 'file:' + self.console_file]
+        subprocess.Popen(cmd,
+                         close_fds=True,
+                         stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+
+        while not os.path.exists(self.pid_file):
+            time.sleep(1)
+
+        with open(self.pid_file, 'r') as pid_f:
+            self.pid = pid_f.readlines()[0].strip()
+
+        if wait:
+            self._wait_for_system(wait_for_cloud_init)
+
+    def console_log(self):
+        if not self.console_file:
+            return b''
+        with open(self.console_file, "rb") as fp:
+            return fp.read()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py
new file mode 100644
index 00000000..85933463
--- /dev/null
+++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py
@@ -0,0 +1,89 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base NoCloud KVM platform."""
+import glob
+import os
+
+from simplestreams import filters
+from simplestreams import mirrors
+from simplestreams import objectstores
+from simplestreams import util as s_util
+
+from ..platforms import Platform
+from .image import NoCloudKVMImage
+from .instance import NoCloudKVMInstance
+from cloudinit import util as c_util
+from tests.cloud_tests import util
+
+
+class NoCloudKVMPlatform(Platform):
+    """NoCloud KVM test platform."""
+
+    platform_name = 'nocloud-kvm'
+
+    def get_image(self, img_conf):
+        """Get image using specified image configuration.
+
+        @param img_conf: configuration for image
+        @return_value: cloud_tests.images instance
+        """
+        (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
+
+        filter = filters.get_filters(['arch=%s' % c_util.get_architecture(),
+                                      'release=%s' % img_conf['release'],
+                                      'ftype=disk1.img'])
+        mirror_config = {'filters': filter,
+                         'keep_items': False,
+                         'max_items': 1,
+                         'checksumming_reader': True,
+                         'item_download': True
+                         }
+
+        def policy(content, path):
+            return s_util.read_signed(content, keyring=img_conf['keyring'])
+
+        smirror = mirrors.UrlMirrorReader(url, policy=policy)
+        tstore = objectstores.FileStore(img_conf['mirror_dir'])
+        tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
+                                             objectstore=tstore)
+        tmirror.sync(smirror, path)
+
+        search_d = os.path.join(img_conf['mirror_dir'], '**',
+                                img_conf['release'], '**', '*.img')
+
+        images = []
+        for fname in glob.iglob(search_d, recursive=True):
+            images.append(fname)
+
+        if len(images) < 1:
+            raise RuntimeError("No images found under '%s'" % search_d)
+        if len(images) > 1:
+            raise RuntimeError(
+                "Multiple images found in '%s': %s" % (search_d,
+                                                       ' '.join(images)))
+
+        image = NoCloudKVMImage(self, img_conf, images[0])
+        return image
+
+    def create_instance(self, properties, config, features,
+                        src_img_path, image_desc=None, use_desc=None,
+                        user_data=None, meta_data=None):
+        """Create an instance
+
+        @param src_img_path: image path to launch from
+        @param properties: image properties
+        @param config: image configuration
+        @param features: image features
+        @param image_desc: description of image being launched
+        @param use_desc: description of container's use
+        @return_value: cloud_tests.instances instance
+        """
+        name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc)
+        img_path = os.path.join(self.config['data_dir'], name + '.qcow2')
+        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
+                    '-b', src_img_path, img_path])
+
+        return NoCloudKVMInstance(self, name, img_path, properties, config,
+                                  features, user_data, meta_data)
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py
new file mode 100644
index 00000000..0005e1f2
--- /dev/null
+++ b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py
@@ -0,0 +1,79 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base NoCloud KVM snapshot."""
+import os
+import shutil
+import tempfile
+
+from ..snapshots import Snapshot
+
+
+class NoCloudKVMSnapshot(Snapshot):
+    """NoCloud KVM image copy backed snapshot."""
+
+    platform_name = "nocloud-kvm"
+
+    def __init__(self, platform, properties, config, features, image_path):
+        """Set up snapshot.
+
+        @param platform: platform object
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        @param image_path: image file to snapshot.
+        """
+        self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot')
+        snapshot = os.path.join(self._workd, 'snapshot')
+        shutil.copyfile(image_path, snapshot)
+        self._image_path = snapshot
+
+        super(NoCloudKVMSnapshot, self).__init__(
+            platform, properties, config, features)
+
+    def launch(self, user_data, meta_data=None, block=True, start=True,
+               use_desc=None):
+        """Launch instance.
+
+        @param user_data: user-data for the instance
+        @param instance_id: instance-id for the instance
+        @param block: wait until instance is created
+        @param start: start instance and wait until fully started
+        @param use_desc: description of snapshot instance use
+        @return_value: an Instance
+        """
+        key_file = os.path.join(self.platform.config['data_dir'],
+                                self.platform.config['public_key'])
+        user_data = self.inject_ssh_key(user_data, key_file)
+
+        instance = self.platform.create_instance(
+            self.properties, self.config, self.features,
+            self._image_path, image_desc=str(self), use_desc=use_desc,
+            user_data=user_data, meta_data=meta_data)
+
+        if start:
+            instance.start()
+
+        return instance
+
+    def inject_ssh_key(self, user_data, key_file):
+        """Inject the authorized key into the user_data."""
+        with open(key_file) as f:
+            value = f.read()
+
+        key = 'ssh_authorized_keys:'
+        value = '  - %s' % value.strip()
+        user_data = user_data.split('\n')
+        if key in user_data:
+            user_data.insert(user_data.index(key) + 1, '%s' % value)
+        else:
+            user_data.insert(-1, '%s' % key)
+            user_data.insert(-1, '%s' % value)
+
+        return '\n'.join(user_data)
+
+    def destroy(self):
+        """Clean up snapshot data."""
+        shutil.rmtree(self._workd)
+        super(NoCloudKVMSnapshot, self).destroy()
+
+# vi: ts=4 expandtab