merge from 7624348712b4502f0085d30c05b34dce3f2ceeae at 17.1-41-g76243487

32 files changed, 548 insertions, 339 deletions

diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index 4a2422ed..71ee7645 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -22,11 +22,21 @@ def collect_script(instance, base_dir, script, script_name):
     """
     LOG.debug('running collect script: %s', script_name)
     (out, err, exit) = instance.run_script(
-        script, rcs=range(0, 256),
+        script.encode(), rcs=False,
         description='collect: {}'.format(script_name))
     c_util.write_file(os.path.join(base_dir, script_name), out)
 
 
+def collect_console(instance, base_dir):
+    LOG.debug('getting console log')
+    try:
+        data = instance.console_log()
+    except NotImplementedError as e:
+        data = 'Not Implemented: %s' % e
+    with open(os.path.join(base_dir, 'console.log'), "wb") as fp:
+        fp.write(data)
+
+
 def collect_test_data(args, snapshot, os_name, test_name):
     """Collect data for test case.
 
@@ -79,8 +89,12 @@ def collect_test_data(args, snapshot, os_name, test_name):
                                          test_output_dir, script, script_name))
                          for script_name, script in test_scripts.items()]
 
+        console_log = partial(
+            run_single, 'collect console',
+            partial(collect_console, instance, test_output_dir))
+
         res = run_stage('collect for test: {}'.format(test_name),
-                        [start_call] + collect_calls)
+                        [start_call] + collect_calls + [console_log])
 
     return res
 
diff --git a/tests/cloud_tests/images/base.py b/tests/cloud_tests/images/base.py
index 0a1e0563..d503108a 100644
--- a/tests/cloud_tests/images/base.py
+++ b/tests/cloud_tests/images/base.py
@@ -2,8 +2,10 @@
 
 """Base class for images."""
 
+from ..util import TargetBase
 
-class Image(object):
+
+class Image(TargetBase):
     """Base class for images."""
 
     platform_name = None
@@ -43,21 +45,6 @@ class Image(object):
         # NOTE: more sophisticated options may be requied at some point
         return self.config.get('setup_overrides', {})
 
-    def execute(self, *args, **kwargs):
-        """Execute command in image, modifying image."""
-        raise NotImplementedError
-
-    def push_file(self, local_path, remote_path):
-        """Copy file at 'local_path' to instance at 'remote_path'."""
-        raise NotImplementedError
-
-    def run_script(self, *args, **kwargs):
-        """Run script in image, modifying image.
-
-        @return_value: script output
-        """
-        raise NotImplementedError
-
     def snapshot(self):
         """Create snapshot of image, block until done."""
         raise NotImplementedError
diff --git a/tests/cloud_tests/images/lxd.py b/tests/cloud_tests/images/lxd.py
index fd4e93c2..5caeba41 100644
--- a/tests/cloud_tests/images/lxd.py
+++ b/tests/cloud_tests/images/lxd.py
@@ -24,7 +24,7 @@ class LXDImage(base.Image):
         @param config: image configuration
         """
         self.modified = False
-        self._instance = None
+        self._img_instance = None
         self._pylxd_image = None
         self.pylxd_image = pylxd_image
         super(LXDImage, self).__init__(platform, config)
@@ -38,9 +38,9 @@ class LXDImage(base.Image):
 
     @pylxd_image.setter
     def pylxd_image(self, pylxd_image):
-        if self._instance:
+        if self._img_instance:
             self._instance.destroy()
-            self._instance = None
+            self._img_instance = None
         if (self._pylxd_image and
                 (self._pylxd_image is not pylxd_image) and
                 (not self.config.get('cache_base_image') or self.modified)):
@@ -49,15 +49,19 @@ class LXDImage(base.Image):
         self._pylxd_image = pylxd_image
 
     @property
-    def instance(self):
-        """Property function."""
-        if not self._instance:
-            self._instance = self.platform.launch_container(
+    def _instance(self):
+        """Internal use only, returns a instance
+
+        This starts an lxc instance from the image, so it is "dirty".
+        Better would be some way to modify this "at rest".
+        lxc-pstart would be an option."""
+        if not self._img_instance:
+            self._img_instance = self.platform.launch_container(
                 self.properties, self.config, self.features,
                 use_desc='image-modification', image_desc=str(self),
                 image=self.pylxd_image.fingerprint)
-        self._instance.start()
-        return self._instance
+            self._img_instance.start()
+        return self._img_instance
 
     @property
     def properties(self):
@@ -144,20 +148,20 @@ class LXDImage(base.Image):
             shutil.rmtree(export_dir)
             shutil.rmtree(extract_dir)
 
-    def execute(self, *args, **kwargs):
+    def _execute(self, *args, **kwargs):
         """Execute command in image, modifying image."""
-        return self.instance.execute(*args, **kwargs)
+        return self._instance._execute(*args, **kwargs)
 
     def push_file(self, local_path, remote_path):
         """Copy file at 'local_path' to instance at 'remote_path'."""
-        return self.instance.push_file(local_path, remote_path)
+        return self._instance.push_file(local_path, remote_path)
 
     def run_script(self, *args, **kwargs):
         """Run script in image, modifying image.
 
         @return_value: script output
         """
-        return self.instance.run_script(*args, **kwargs)
+        return self._instance.run_script(*args, **kwargs)
 
     def snapshot(self):
         """Create snapshot of image, block until done."""
@@ -169,7 +173,7 @@ class LXDImage(base.Image):
         # clone current instance
         instance = self.platform.launch_container(
             self.properties, self.config, self.features,
-            container=self.instance.name, image_desc=str(self),
+            container=self._instance.name, image_desc=str(self),
             use_desc='snapshot', container_config=conf)
         # wait for cloud-init before boot_clean_script is run to ensure
         # /var/lib/cloud is removed cleanly
diff --git a/tests/cloud_tests/images/nocloudkvm.py b/tests/cloud_tests/images/nocloudkvm.py
index a7af0e59..1e7962cb 100644
--- a/tests/cloud_tests/images/nocloudkvm.py
+++ b/tests/cloud_tests/images/nocloudkvm.py
@@ -2,6 +2,8 @@
 
 """NoCloud KVM Image Base Class."""
 
+from cloudinit import util as c_util
+
 from tests.cloud_tests.images import base
 from tests.cloud_tests.snapshots import nocloudkvm as nocloud_kvm_snapshot
 
@@ -19,24 +21,11 @@ class NoCloudKVMImage(base.Image):
         @param img_path: path to the image
         """
         self.modified = False
-        self._instance = None
         self._img_path = img_path
 
         super(NoCloudKVMImage, self).__init__(platform, config)
 
     @property
-    def instance(self):
-        """Returns an instance of an image."""
-        if not self._instance:
-            if not self._img_path:
-                raise RuntimeError()
-
-            self._instance = self.platform.create_image(
-                self.properties, self.config, self.features, self._img_path,
-                image_desc=str(self), use_desc='image-modification')
-        return self._instance
-
-    @property
     def properties(self):
         """Dictionary containing: 'arch', 'os', 'version', 'release'."""
         return {
@@ -46,20 +35,26 @@ class NoCloudKVMImage(base.Image):
             'version': self.config['version'],
         }
 
-    def execute(self, *args, **kwargs):
+    def _execute(self, command, stdin=None, env=None):
         """Execute command in image, modifying image."""
-        return self.instance.execute(*args, **kwargs)
+        return self.mount_image_callback(command, stdin=stdin, env=env)
 
-    def push_file(self, local_path, remote_path):
-        """Copy file at 'local_path' to instance at 'remote_path'."""
-        return self.instance.push_file(local_path, remote_path)
+    def mount_image_callback(self, command, stdin=None, env=None):
+        """Run mount-image-callback."""
 
-    def run_script(self, *args, **kwargs):
-        """Run script in image, modifying image.
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
 
-        @return_value: script output
-        """
-        return self.instance.run_script(*args, **kwargs)
+        mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts',
+                      '--system-resolvconf', self._img_path,
+                      '--', 'chroot', '_MOUNTPOINT_']
+        try:
+            out, err = c_util.subp(mic_chroot + env_args + list(command),
+                                   data=stdin, decode=False)
+            return (out, err, 0)
+        except c_util.ProcessExecutionError as e:
+            return (e.stdout, e.stderr, e.exit_code)
 
     def snapshot(self):
         """Create snapshot of image, block until done."""
@@ -82,7 +77,6 @@ class NoCloudKVMImage(base.Image):
         framework decide whether to keep or destroy everything.
         """
         self._img_path = None
-        self._instance.destroy()
         super(NoCloudKVMImage, self).destroy()
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py
index 9bdda608..8c59d62c 100644
--- a/tests/cloud_tests/instances/base.py
+++ b/tests/cloud_tests/instances/base.py
@@ -2,8 +2,10 @@
 
 """Base instance."""
 
+from ..util import TargetBase
 
-class Instance(object):
+
+class Instance(TargetBase):
     """Base instance object."""
 
     platform_name = None
@@ -22,82 +24,7 @@ class Instance(object):
         self.properties = properties
         self.config = config
         self.features = features
-
-    def execute(self, command, stdout=None, stderr=None, env=None,
-                rcs=None, description=None):
-        """Execute command in instance, recording output, error and exit code.
-
-        Assumes functional networking and execution as root with the
-        target filesystem being available at /.
-
-        @param command: the command to execute as root inside the image
-            if command is a string, then it will be executed as:
-            ['sh', '-c', command]
-        @param stdout, stderr: file handles to write output and error to
-        @param env: environment variables
-        @param rcs: allowed return codes from command
-        @param description: purpose of command
-        @return_value: tuple containing stdout data, stderr data, exit code
-        """
-        raise NotImplementedError
-
-    def read_data(self, remote_path, decode=False):
-        """Read data from instance filesystem.
-
-        @param remote_path: path in instance
-        @param decode: return as string
-        @return_value: data as str or bytes
-        """
-        raise NotImplementedError
-
-    def write_data(self, remote_path, data):
-        """Write data to instance filesystem.
-
-        @param remote_path: path in instance
-        @param data: data to write, either str or bytes
-        """
-        raise NotImplementedError
-
-    def pull_file(self, remote_path, local_path):
-        """Copy file at 'remote_path', from instance to 'local_path'.
-
-        @param remote_path: path on remote instance
-        @param local_path: path on local instance
-        """
-        with open(local_path, 'wb') as fp:
-            fp.write(self.read_data(remote_path))
-
-    def push_file(self, local_path, remote_path):
-        """Copy file at 'local_path' to instance at 'remote_path'.
-
-        @param local_path: path on local instance
-        @param remote_path: path on remote instance
-        """
-        with open(local_path, 'rb') as fp:
-            self.write_data(remote_path, fp.read())
-
-    def run_script(self, script, rcs=None, description=None):
-        """Run script in target and return stdout.
-
-        @param script: script contents
-        @param rcs: allowed return codes from script
-        @param description: purpose of script
-        @return_value: stdout from script
-        """
-        script_path = self.tmpfile()
-        try:
-            self.write_data(script_path, script)
-            return self.execute(
-                ['/bin/bash', script_path], rcs=rcs, description=description)
-        finally:
-            self.execute(['rm', '-f', script_path], rcs=rcs)
-
-    def tmpfile(self):
-        """Get a tmp file in the target.
-
-        @return_value: path to new file in target
-        """
-        return self.execute(['mktemp'])[0].strip()
+        self._tmp_count = 0
 
     def console_log(self):
         """Instance console.
diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py
index a43918c2..3b035d86 100644
--- a/tests/cloud_tests/instances/lxd.py
+++ b/tests/cloud_tests/instances/lxd.py
@@ -2,8 +2,11 @@
 
 """Base LXD instance."""
 
-from tests.cloud_tests.instances import base
-from tests.cloud_tests import util
+from . import base
+
+import os
+import shutil
+from tempfile import mkdtemp
 
 
 class LXDInstance(base.Instance):
@@ -24,6 +27,8 @@ class LXDInstance(base.Instance):
         self._pylxd_container = pylxd_container
         super(LXDInstance, self).__init__(
             platform, name, properties, config, features)
+        self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name))
+        self._setup_console_log()
 
     @property
     def pylxd_container(self):
@@ -31,74 +36,69 @@ class LXDInstance(base.Instance):
         self._pylxd_container.sync()
         return self._pylxd_container
 
-    def execute(self, command, stdout=None, stderr=None, env=None,
-                rcs=None, description=None):
-        """Execute command in instance, recording output, error and exit code.
-
-        Assumes functional networking and execution as root with the
-        target filesystem being available at /.
-
-        @param command: the command to execute as root inside the image
-            if command is a string, then it will be executed as:
-            ['sh', '-c', command]
-        @param stdout: file handler to write output
-        @param stderr: file handler to write error
-        @param env: environment variables
-        @param rcs: allowed return codes from command
-        @param description: purpose of command
-        @return_value: tuple containing stdout data, stderr data, exit code
-        """
+    def _setup_console_log(self):
+        logf = os.path.join(self.tmpd, "console.log")
+
+        # doing this ensures we can read it. Otherwise it ends up root:root.
+        with open(logf, "w") as fp:
+            fp.write("# %s\n" % self.name)
+
+        cfg = "lxc.console.logfile=%s" % logf
+        orig = self._pylxd_container.config.get('raw.lxc', "")
+        if orig:
+            orig += "\n"
+        self._pylxd_container.config['raw.lxc'] = orig + cfg
+        self._pylxd_container.save()
+        self._console_log_file = logf
+
+    def _execute(self, command, stdin=None, env=None):
         if env is None:
             env = {}
 
-        if isinstance(command, str):
-            command = ['sh', '-c', command]
+        if stdin is not None:
+            # pylxd does not support input to execute.
+            # https://github.com/lxc/pylxd/issues/244
+            #
+            # The solution here is write a tmp file in the container
+            # and then execute a shell that sets it standard in to
+            # be from that file, removes it, and calls the comand.
+            tmpf = self.tmpfile()
+            self.write_data(tmpf, stdin)
+            ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"'
+            command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] +
+                       list(command))
 
         # ensure instance is running and execute the command
         self.start()
+        # execute returns a ContainerExecuteResult, named tuple
+        # (exit_code, stdout, stderr)
         res = self.pylxd_container.execute(command, environment=env)
 
         # get out, exit and err from pylxd return
-        if hasattr(res, 'exit_code'):
-            # pylxd 2.2 returns ContainerExecuteResult, named tuple of
-            # (exit_code, out, err)
-            (exit, out, err) = res
-        else:
+        if not hasattr(res, 'exit_code'):
             # pylxd 2.1.3 and earlier only return out and err, no exit
-            # LOG.warning('using pylxd version < 2.2')
-            (out, err) = res
-            exit = 0
-
-        # write data to file descriptors if needed
-        if stdout:
-            stdout.write(out)
-        if stderr:
-            stderr.write(err)
-
-        # if the command exited with a code not allowed in rcs, then fail
-        if exit not in (rcs if rcs else (0,)):
-            error_desc = ('Failed command to: {}'.format(description)
-                          if description else None)
-            raise util.InTargetExecuteError(
-                out, err, exit, command, self.name, error_desc)
+            raise RuntimeError(
+                "No 'exit_code' in pylxd.container.execute return.\n"
+                "pylxd > 2.2 is required.")
 
-        return (out, err, exit)
+        return res.stdout, res.stderr, res.exit_code
 
     def read_data(self, remote_path, decode=False):
         """Read data from instance filesystem.
 
         @param remote_path: path in instance
-        @param decode: return as string
-        @return_value: data as str or bytes
+        @param decode: decode data before returning.
+        @return_value: content of remote_path as bytes if 'decode' is False,
+                       and as string if 'decode' is True.
         """
         data = self.pylxd_container.files.get(remote_path)
-        return data.decode() if decode and isinstance(data, bytes) else data
+        return data.decode() if decode else data
 
     def write_data(self, remote_path, data):
         """Write data to instance filesystem.
 
         @param remote_path: path in instance
-        @param data: data to write, either str or bytes
+        @param data: data to write in bytes
         """
         self.pylxd_container.files.put(remote_path, data)
 
@@ -107,7 +107,14 @@ class LXDInstance(base.Instance):
 
         @return_value: bytes of this instance’s console
         """
-        raise NotImplementedError
+        if not os.path.exists(self._console_log_file):
+            raise NotImplementedError(
+                "Console log '%s' does not exist. If this is a remote "
+                "lxc, then this is really NotImplementedError.  If it is "
+                "A local lxc, then this is a RuntimeError."
+                "https://github.com/lxc/lxd/issues/1129")
+        with open(self._console_log_file, "rb") as fp:
+            return fp.read()
 
     def reboot(self, wait=True):
         """Reboot instance."""
@@ -144,6 +151,7 @@ class LXDInstance(base.Instance):
         if self.platform.container_exists(self.name):
             raise OSError('container {} was not properly removed'
                           .format(self.name))
+        shutil.rmtree(self.tmpd)
         super(LXDInstance, self).destroy()
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/instances/nocloudkvm.py b/tests/cloud_tests/instances/nocloudkvm.py
index 8a0e5319..cc825800 100644
--- a/tests/cloud_tests/instances/nocloudkvm.py
+++ b/tests/cloud_tests/instances/nocloudkvm.py
@@ -12,11 +12,18 @@ from cloudinit import util as c_util
 from tests.cloud_tests.instances import base
 from tests.cloud_tests import util
 
+# This domain contains reverse lookups for hostnames that are used.
+# The primary reason is so sudo will return quickly when it attempts
+# to look up the hostname.  i9n is just short for 'integration'.
+# see also bug 1730744 for why we had to do this.
+CI_DOMAIN = "i9n.cloud-init.io"
+
 
 class NoCloudKVMInstance(base.Instance):
     """NoCloud KVM backed instance."""
 
     platform_name = "nocloud-kvm"
+    _ssh_client = None
 
     def __init__(self, platform, name, properties, config, features,
                  user_data, meta_data):
@@ -35,6 +42,7 @@ class NoCloudKVMInstance(base.Instance):
         self.ssh_port = None
         self.pid = None
         self.pid_file = None
+        self.console_file = None
 
         super(NoCloudKVMInstance, self).__init__(
             platform, name, properties, config, features)
@@ -51,43 +59,18 @@ class NoCloudKVMInstance(base.Instance):
             os.remove(self.pid_file)
 
         self.pid = None
-        super(NoCloudKVMInstance, self).destroy()
-
-    def execute(self, command, stdout=None, stderr=None, env=None,
-                rcs=None, description=None):
-        """Execute command in instance.
-
-        Assumes functional networking and execution as root with the
-        target filesystem being available at /.
-
-        @param command: the command to execute as root inside the image
-            if command is a string, then it will be executed as:
-            ['sh', '-c', command]
-        @param stdout, stderr: file handles to write output and error to
-        @param env: environment variables
-        @param rcs: allowed return codes from command
-        @param description: purpose of command
-        @return_value: tuple containing stdout data, stderr data, exit code
-        """
-        if env is None:
-            env = {}
-
-        if isinstance(command, str):
-            command = ['sh', '-c', command]
+        if self._ssh_client:
+            self._ssh_client.close()
+            self._ssh_client = None
 
-        if self.pid:
-            return self.ssh(command)
-        else:
-            return self.mount_image_callback(command) + (0,)
+        super(NoCloudKVMInstance, self).destroy()
 
-    def mount_image_callback(self, cmd):
-        """Run mount-image-callback."""
-        out, err = c_util.subp(['sudo', 'mount-image-callback',
-                                '--system-mounts', '--system-resolvconf',
-                                self.name, '--', 'chroot',
-                                '_MOUNTPOINT_'] + cmd)
+    def _execute(self, command, stdin=None, env=None):
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
 
-        return out, err
+        return self.ssh(['sudo'] + env_args + list(command), stdin=stdin)
 
     def generate_seed(self, tmpdir):
         """Generate nocloud seed from user-data"""
@@ -109,57 +92,31 @@ class NoCloudKVMInstance(base.Instance):
         s.close()
         return num
 
-    def push_file(self, local_path, remote_path):
-        """Copy file at 'local_path' to instance at 'remote_path'.
-
-        If we have a pid then SSH is up, otherwise, use
-        mount-image-callback.
-
-        @param local_path: path on local instance
-        @param remote_path: path on remote instance
-        """
-        if self.pid:
-            super(NoCloudKVMInstance, self).push_file()
-        else:
-            local_file = open(local_path)
-            p = subprocess.Popen(['sudo', 'mount-image-callback',
-                                  '--system-mounts', '--system-resolvconf',
-                                  self.name, '--', 'chroot', '_MOUNTPOINT_',
-                                  '/bin/sh', '-c', 'cat - > %s' % remote_path],
-                                 stdin=local_file,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-            p.wait()
-
-    def sftp_put(self, path, data):
-        """SFTP put a file."""
-        client = self._ssh_connect()
-        sftp = client.open_sftp()
-
-        with sftp.open(path, 'w') as f:
-            f.write(data)
-
-        client.close()
-
-    def ssh(self, command):
+    def ssh(self, command, stdin=None):
         """Run a command via SSH."""
         client = self._ssh_connect()
 
+        cmd = util.shell_pack(command)
         try:
-            _, out, err = client.exec_command(util.shell_pack(command))
-        except paramiko.SSHException:
-            raise util.InTargetExecuteError('', '', -1, command, self.name)
-
-        exit = out.channel.recv_exit_status()
-        out = ''.join(out.readlines())
-        err = ''.join(err.readlines())
-        client.close()
-
-        return out, err, exit
+            fp_in, fp_out, fp_err = client.exec_command(cmd)
+            channel = fp_in.channel
+            if stdin is not None:
+                fp_in.write(stdin)
+                fp_in.close()
+
+            channel.shutdown_write()
+            rc = channel.recv_exit_status()
+            return (fp_out.read(), fp_err.read(), rc)
+        except paramiko.SSHException as e:
+            raise util.InTargetExecuteError(
+                b'', b'', -1, command, self.name, reason=e)
 
     def _ssh_connect(self, hostname='localhost', username='ubuntu',
                      banner_timeout=120, retry_attempts=30):
         """Connect via SSH."""
+        if self._ssh_client:
+            return self._ssh_client
+
         private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file)
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
@@ -168,6 +125,7 @@ class NoCloudKVMInstance(base.Instance):
                 client.connect(hostname=hostname, username=username,
                                port=self.ssh_port, pkey=private_key,
                                banner_timeout=banner_timeout)
+                self._ssh_client = client
                 return client
             except (paramiko.SSHException, TypeError):
                 time.sleep(1)
@@ -183,15 +141,19 @@ class NoCloudKVMInstance(base.Instance):
         tmpdir = self.platform.config['data_dir']
         seed = self.generate_seed(tmpdir)
         self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name)
+        self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name)
         self.ssh_port = self.get_free_port()
 
-        subprocess.Popen(['./tools/xkvm',
-                          '--disk', '%s,cache=unsafe' % self.name,
-                          '--disk', '%s,cache=unsafe' % seed,
-                          '--netdev',
-                          'user,hostfwd=tcp::%s-:22' % self.ssh_port,
-                          '--', '-pidfile', self.pid_file, '-vnc', 'none',
-                          '-m', '2G', '-smp', '2'],
+        cmd = ['./tools/xkvm',
+               '--disk', '%s,cache=unsafe' % self.name,
+               '--disk', '%s,cache=unsafe' % seed,
+               '--netdev', ','.join(['user',
+                                     'hostfwd=tcp::%s-:22' % self.ssh_port,
+                                     'dnssearch=%s' % CI_DOMAIN]),
+               '--', '-pidfile', self.pid_file, '-vnc', 'none',
+               '-m', '2G', '-smp', '2', '-nographic',
+               '-serial', 'file:' + self.console_file]
+        subprocess.Popen(cmd,
                          close_fds=True,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,
@@ -206,12 +168,10 @@ class NoCloudKVMInstance(base.Instance):
         if wait:
             self._wait_for_system(wait_for_cloud_init)
 
-    def write_data(self, remote_path, data):
-        """Write data to instance filesystem.
-
-        @param remote_path: path in instance
-        @param data: data to write, either str or bytes
-        """
-        self.sftp_put(remote_path, data)
+    def console_log(self):
+        if not self.console_file:
+            return b''
+        with open(self.console_file, "rb") as fp:
+            return fp.read()
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/examples/run_commands.yaml b/tests/cloud_tests/testcases/examples/run_commands.yaml
index b0e311ba..f80eb8ce 100644
--- a/tests/cloud_tests/testcases/examples/run_commands.yaml
+++ b/tests/cloud_tests/testcases/examples/run_commands.yaml
@@ -7,10 +7,10 @@ enabled: False
 cloud_config: |
   #cloud-config
   runcmd:
-   - echo cloud-init run cmd test > /tmp/run_cmd
+   - echo cloud-init run cmd test > /var/tmp/run_cmd
 collect_scripts:
   run_cmd: |
     #!/bin/bash
-    cat /tmp/run_cmd
+    cat /var/tmp/run_cmd
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py
index d299e9ad..dfbdeadf 100644
--- a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py
+++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py
@@ -11,13 +11,13 @@ class TestAptconfigureSourcesPPA(base.CloudTestCase):
         """Test specific ppa added."""
         out = self.get_data_file('sources.list')
         self.assertIn(
-            'http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu', out)
+            'http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu', out)
 
     def test_ppa_key(self):
         """Test ppa key added."""
         out = self.get_data_file('apt-key')
         self.assertIn(
-            '1BC3 0F71 5A3B 8612 47A8  1A5E 55FE 7C8C 0165 013E', out)
-        self.assertIn('Launchpad PPA for curtin developers', out)
+            '1FF0 D853 5EF7 E719 E5C8  1B9C 083D 06FB E4D3 04DF', out)
+        self.assertIn('Launchpad PPA for cloud init development team', out)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml
index 9efdae52..b997bcfb 100644
--- a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml
+++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml
@@ -2,7 +2,7 @@
 # Add a PPA to source.list
 #
 # NOTE: on older ubuntu releases the sources file added is named
-#       'curtin-dev-test-archive-trusty', without 'ubuntu' in the middle
+#       'cloud-init-dev-test-archive-trusty', without 'ubuntu' in the middle
 required_features:
   - apt
   - ppa
@@ -14,11 +14,11 @@ cloud_config: |
       source1:
         keyid: 0165013E
         keyserver: keyserver.ubuntu.com
-        source: "ppa:curtin-dev/test-archive"
+        source: "ppa:cloud-init-dev/test-archive"
 collect_scripts:
   sources.list: |
     #!/bin/bash
-    cat /etc/apt/sources.list.d/curtin-dev-ubuntu-test-archive-*.list
+    cat /etc/apt/sources.list.d/cloud-init-dev-ubuntu-test-archive-*.list
   apt-key: |
     #!/bin/bash
     apt-key finger
diff --git a/tests/cloud_tests/testcases/modules/keys_to_console.py b/tests/cloud_tests/testcases/modules/keys_to_console.py
index 88b6812e..07f38112 100644
--- a/tests/cloud_tests/testcases/modules/keys_to_console.py
+++ b/tests/cloud_tests/testcases/modules/keys_to_console.py
@@ -10,13 +10,13 @@ class TestKeysToConsole(base.CloudTestCase):
     def test_excluded_keys(self):
         """Test excluded keys missing."""
         out = self.get_data_file('syslog')
-        self.assertNotIn('DSA', out)
-        self.assertNotIn('ECDSA', out)
+        self.assertNotIn('(DSA)', out)
+        self.assertNotIn('(ECDSA)', out)
 
     def test_expected_keys(self):
         """Test expected keys exist."""
         out = self.get_data_file('syslog')
-        self.assertIn('ED25519', out)
-        self.assertIn('RSA', out)
+        self.assertIn('(ED25519)', out)
+        self.assertIn('(RSA)', out)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/runcmd.yaml b/tests/cloud_tests/testcases/modules/runcmd.yaml
index 04e5a050..8309a883 100644
--- a/tests/cloud_tests/testcases/modules/runcmd.yaml
+++ b/tests/cloud_tests/testcases/modules/runcmd.yaml
@@ -4,10 +4,10 @@
 cloud_config: |
   #cloud-config
   runcmd:
-   - echo cloud-init run cmd test > /tmp/run_cmd
+   - echo cloud-init run cmd test > /var/tmp/run_cmd
 collect_scripts:
   run_cmd: |
     #!/bin/bash
-    cat /tmp/run_cmd
+    cat /var/tmp/run_cmd
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/set_hostname.py b/tests/cloud_tests/testcases/modules/set_hostname.py
index 6e96a75c..1dbe64c2 100644
--- a/tests/cloud_tests/testcases/modules/set_hostname.py
+++ b/tests/cloud_tests/testcases/modules/set_hostname.py
@@ -7,9 +7,11 @@ from tests.cloud_tests.testcases import base
 class TestHostname(base.CloudTestCase):
     """Test hostname module."""
 
+    ex_hostname = "cloudinit2"
+
     def test_hostname(self):
         """Test hostname command shows correct output."""
         out = self.get_data_file('hostname')
-        self.assertIn('myhostname', out)
+        self.assertIn(self.ex_hostname, out)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/set_hostname.yaml b/tests/cloud_tests/testcases/modules/set_hostname.yaml
index c96344cf..071fb220 100644
--- a/tests/cloud_tests/testcases/modules/set_hostname.yaml
+++ b/tests/cloud_tests/testcases/modules/set_hostname.yaml
@@ -5,7 +5,8 @@ required_features:
   - hostname
 cloud_config: |
   #cloud-config
-  hostname: myhostname
+  hostname: cloudinit2
+
 collect_scripts:
   hosts: |
     #!/bin/bash
diff --git a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py
index 398f3d40..eb6f0650 100644
--- a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py
+++ b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py
@@ -1,26 +1,31 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """cloud-init Integration Test Verify Script."""
+from tests.cloud_tests.instances.nocloudkvm import CI_DOMAIN
 from tests.cloud_tests.testcases import base
 
 
 class TestHostnameFqdn(base.CloudTestCase):
     """Test Hostname module."""
 
+    ex_hostname = "cloudinit1"
+    ex_fqdn = "cloudinit2." + CI_DOMAIN
+
     def test_hostname(self):
         """Test hostname output."""
         out = self.get_data_file('hostname')
-        self.assertIn('myhostname', out)
+        self.assertIn(self.ex_hostname, out)
 
     def test_hostname_fqdn(self):
         """Test hostname fqdn output."""
         out = self.get_data_file('fqdn')
-        self.assertIn('host.myorg.com', out)
+        self.assertIn(self.ex_fqdn, out)
 
     def test_hosts(self):
         """Test /etc/hosts file."""
         out = self.get_data_file('hosts')
-        self.assertIn('127.0.1.1 host.myorg.com myhostname', out)
+        self.assertIn('127.0.1.1 %s %s' % (self.ex_fqdn, self.ex_hostname),
+                      out)
         self.assertIn('127.0.0.1 localhost', out)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml
index daf75931..a85ee79e 100644
--- a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml
+++ b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml
@@ -6,8 +6,9 @@ required_features:
 cloud_config: |
   #cloud-config
   manage_etc_hosts: true
-  hostname: myhostname
-  fqdn: host.myorg.com
+  hostname: cloudinit1
+  # this needs changing if CI_DOMAIN were updated.
+  fqdn: cloudinit2.i9n.cloud-init.io
 collect_scripts:
   hosts: |
     #!/bin/bash
diff --git a/tests/cloud_tests/testcases/modules/set_password_expire.py b/tests/cloud_tests/testcases/modules/set_password_expire.py
index a1c3aa08..967aca7b 100644
--- a/tests/cloud_tests/testcases/modules/set_password_expire.py
+++ b/tests/cloud_tests/testcases/modules/set_password_expire.py
@@ -18,6 +18,6 @@ class TestPasswordExpire(base.CloudTestCase):
     def test_sshd_config(self):
         """Test sshd config allows passwords."""
         out = self.get_data_file('sshd_config')
-        self.assertIn('PasswordAuthentication no', out)
+        self.assertIn('PasswordAuthentication yes', out)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/set_password_expire.yaml b/tests/cloud_tests/testcases/modules/set_password_expire.yaml
index 789604b0..ba6344b9 100644
--- a/tests/cloud_tests/testcases/modules/set_password_expire.yaml
+++ b/tests/cloud_tests/testcases/modules/set_password_expire.yaml
@@ -6,7 +6,9 @@ required_features:
 cloud_config: |
   #cloud-config
   chpasswd: { expire: True }
+  ssh_pwauth: yes
   users:
+    - default
     - name: tom
       password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE.
       lock_passwd: false
diff --git a/tests/cloud_tests/testcases/modules/set_password_list.yaml b/tests/cloud_tests/testcases/modules/set_password_list.yaml
index a2a89c9d..fd3e1e44 100644
--- a/tests/cloud_tests/testcases/modules/set_password_list.yaml
+++ b/tests/cloud_tests/testcases/modules/set_password_list.yaml
@@ -5,6 +5,7 @@ cloud_config: |
   #cloud-config
   ssh_pwauth: yes
   users:
+    - default
     - name: tom
       # md5 gotomgo
       passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0"
diff --git a/tests/cloud_tests/testcases/modules/set_password_list_string.yaml b/tests/cloud_tests/testcases/modules/set_password_list_string.yaml
index c2a0f631..e9fe54b0 100644
--- a/tests/cloud_tests/testcases/modules/set_password_list_string.yaml
+++ b/tests/cloud_tests/testcases/modules/set_password_list_string.yaml
@@ -5,6 +5,7 @@ cloud_config: |
   #cloud-config
   ssh_pwauth: yes
   users:
+    - default
     - name: tom
       # md5 gotomgo
       passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0"
diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py
index 82223217..e7329d48 100644
--- a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py
+++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py
@@ -13,12 +13,4 @@ class TestSshKeyFingerprintsDisable(base.CloudTestCase):
         self.assertIn('Skipping module named ssh-authkey-fingerprints, '
                       'logging of ssh fingerprints disabled', out)
 
-    def test_syslog(self):
-        """Verify output of syslog."""
-        out = self.get_data_file('syslog')
-        self.assertNotRegex(out, r'256 SHA256:.*(ECDSA)')
-        self.assertNotRegex(out, r'256 SHA256:.*(ED25519)')
-        self.assertNotRegex(out, r'1024 SHA256:.*(DSA)')
-        self.assertNotRegex(out, r'2048 SHA256:.*(RSA)')
-
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml
index 746653ec..d93893e2 100644
--- a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml
+++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml
@@ -5,7 +5,6 @@ required_features:
   - syslog
 cloud_config: |
   #cloud-config
-  ssh_genkeytypes: []
   no_ssh_fingerprints: true
 collect_scripts:
   syslog: |
diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_generate.py b/tests/cloud_tests/testcases/modules/ssh_keys_generate.py
index fd6d9ba5..b68f5565 100644
--- a/tests/cloud_tests/testcases/modules/ssh_keys_generate.py
+++ b/tests/cloud_tests/testcases/modules/ssh_keys_generate.py
@@ -9,11 +9,6 @@ class TestSshKeysGenerate(base.CloudTestCase):
 
     # TODO: Check cloud-init-output for the correct keys being generated
 
-    def test_ubuntu_authorized_keys(self):
-        """Test passed in key is not in list for ubuntu."""
-        out = self.get_data_file('auth_keys_ubuntu')
-        self.assertEqual('', out)
-
     def test_dsa_public(self):
         """Test dsa public key not generated."""
         out = self.get_data_file('dsa_public')
diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml b/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml
index 659fd939..0a7adf62 100644
--- a/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml
+++ b/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml
@@ -10,12 +10,6 @@ cloud_config: |
   - ed25519
   authkey_hash: sha512
 collect_scripts:
-  auth_keys_root: |
-    #!/bin/bash
-    cat /root/.ssh/authorized_keys
-  auth_keys_ubuntu: |
-    #!/bin/bash
-    cat /home/ubuntu/ssh/authorized_keys
   dsa_public: |
     #!/bin/bash
     cat /etc/ssh/ssh_host_dsa_key.pub
diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_provided.py b/tests/cloud_tests/testcases/modules/ssh_keys_provided.py
index 544649da..add3f469 100644
--- a/tests/cloud_tests/testcases/modules/ssh_keys_provided.py
+++ b/tests/cloud_tests/testcases/modules/ssh_keys_provided.py
@@ -7,17 +7,6 @@ from tests.cloud_tests.testcases import base
 class TestSshKeysProvided(base.CloudTestCase):
     """Test ssh keys module."""
 
-    def test_ubuntu_authorized_keys(self):
-        """Test passed in key is not in list for ubuntu."""
-        out = self.get_data_file('auth_keys_ubuntu')
-        self.assertEqual('', out)
-
-    def test_root_authorized_keys(self):
-        """Test passed in key is in authorized list for root."""
-        out = self.get_data_file('auth_keys_root')
-        self.assertIn('lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs50'
-                      '6oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w==', out)
-
     def test_dsa_public(self):
         """Test dsa public key passed in."""
         out = self.get_data_file('dsa_public')
diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml b/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml
index 5ceb3623..41f63550 100644
--- a/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml
+++ b/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml
@@ -71,12 +71,6 @@ cloud_config: |
       -----END EC PRIVATE KEY-----
     ecdsa_public: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFsS5Tvky/IC/dXhE/afxxUG6kdQOvdQJCYGZN42OZqWasYF+L3IG+3/wrV7jOrNrL3AyagHl6+lpPDiSXDMcpQ= root@xenial-lxd
 collect_scripts:
-  auth_keys_root: |
-    #!/bin/bash
-    cat /root/.ssh/authorized_keys
-  auth_keys_ubuntu: |
-    #!/bin/bash
-    cat /home/ubuntu/ssh/authorized_keys
   dsa_public: |
     #!/bin/bash
     cat /etc/ssh/ssh_host_dsa_key.pub
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 4357fbb0..c5cd6974 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -7,6 +7,7 @@ import copy
 import glob
 import os
 import random
+import shlex
 import shutil
 import string
 import subprocess
@@ -285,20 +286,165 @@ def shell_pack(cmd):
     return 'eval set -- "$(echo %s | base64 --decode)" && exec "$@"' % b64
 
 
+def shell_quote(cmd):
+    if isinstance(cmd, (tuple, list)):
+        return ' '.join([shlex.quote(x) for x in cmd])
+    return shlex.quote(cmd)
+
+
+class TargetBase(object):
+    _tmp_count = 0
+
+    def execute(self, command, stdin=None, env=None,
+                rcs=None, description=None):
+        """Execute command in instance, recording output, error and exit code.
+
+        Assumes functional networking and execution as root with the
+        target filesystem being available at /.
+
+        @param command: the command to execute as root inside the image
+            if command is a string, then it will be executed as:
+            ['sh', '-c', command]
+        @param stdin: bytes content for standard in
+        @param env: environment variables
+        @param rcs: return codes.
+                    None (default): non-zero exit code will raise exception.
+                    False: any is allowed (No execption raised).
+                    list of int: any rc not in the list will raise exception.
+        @param description: purpose of command
+        @return_value: tuple containing stdout data, stderr data, exit code
+        """
+        if isinstance(command, str):
+            command = ['sh', '-c', command]
+
+        if rcs is None:
+            rcs = (0,)
+
+        if description:
+            LOG.debug('Executing "%s"', description)
+        else:
+            LOG.debug("Executing command: %s", shell_quote(command))
+
+        out, err, rc = self._execute(command=command, stdin=stdin, env=env)
+
+        # False means accept anything.
+        if (rcs is False or rc in rcs):
+            return out, err, rc
+
+        raise InTargetExecuteError(out, err, rc, command, description)
+
+    def _execute(self, command, stdin=None, env=None):
+        """Execute command in inside, return stdout, stderr and exit code.
+
+        Assumes functional networking and execution as root with the
+        target filesystem being available at /.
+
+        @param stdin: bytes content for standard in
+        @param env: environment variables
+        @return_value: tuple containing stdout data, stderr data, exit code
+
+        This is intended to be implemented by the Image or Instance.
+        Many callers will use the higher level 'execute'."""
+        raise NotImplementedError("_execute must be implemented by subclass.")
+
+    def read_data(self, remote_path, decode=False):
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: decode data before returning.
+        @return_value: content of remote_path as bytes if 'decode' is False,
+                       and as string if 'decode' is True.
+        """
+        # when sh is invoked with '-c', then the first argument is "$0"
+        # which is commonly understood as the "program name".
+        # 'read_data' is the program name, and 'remote_path' is '$1'
+        stdout, stderr, rc = self._execute(
+            ["sh", "-c", 'exec cat "$1"', 'read_data', remote_path])
+        if rc != 0:
+            raise RuntimeError("Failed to read file '%s'" % remote_path)
+
+        if decode:
+            return stdout.decode()
+        return stdout
+
+    def write_data(self, remote_path, data):
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write in bytes
+        """
+        # when sh is invoked with '-c', then the first argument is "$0"
+        # which is commonly understood as the "program name".
+        # 'write_data' is the program name, and 'remote_path' is '$1'
+        _, _, rc = self._execute(
+            ["sh", "-c", 'exec cat >"$1"', 'write_data', remote_path],
+            stdin=data)
+
+        if rc != 0:
+            raise RuntimeError("Failed to write to '%s'" % remote_path)
+        return
+
+    def pull_file(self, remote_path, local_path):
+        """Copy file at 'remote_path', from instance to 'local_path'.
+
+        @param remote_path: path on remote instance
+        @param local_path: path on local instance
+        """
+        with open(local_path, 'wb') as fp:
+            fp.write(self.read_data(remote_path))
+
+    def push_file(self, local_path, remote_path):
+        """Copy file at 'local_path' to instance at 'remote_path'.
+
+        @param local_path: path on local instance
+        @param remote_path: path on remote instance"""
+        with open(local_path, "rb") as fp:
+            self.write_data(remote_path, data=fp.read())
+
+    def run_script(self, script, rcs=None, description=None):
+        """Run script in target and return stdout.
+
+        @param script: script contents
+        @param rcs: allowed return codes from script
+        @param description: purpose of script
+        @return_value: stdout from script
+        """
+        # Just write to a file, add execute, run it, then remove it.
+        shblob = '; '.join((
+            'set -e',
+            's="$1"',
+            'shift',
+            'cat > "$s"',
+            'trap "rm -f $s" EXIT',
+            'chmod +x "$s"',
+            '"$s" "$@"'))
+        return self.execute(
+            ['sh', '-c', shblob, 'runscript', self.tmpfile()],
+            stdin=script, description=description, rcs=rcs)
+
+    def tmpfile(self):
+        """Get a tmp file in the target.
+
+        @return_value: path to new file in target
+        """
+        path = "/tmp/%s-%04d" % (type(self).__name__, self._tmp_count)
+        self._tmp_count += 1
+        return path
+
+
 class InTargetExecuteError(c_util.ProcessExecutionError):
     """Error type for in target commands that fail."""
 
-    default_desc = 'Unexpected error while running command in target instance'
+    default_desc = 'Unexpected error while running command.'
 
-    def __init__(self, stdout, stderr, exit_code, cmd, instance,
-                 description=None):
+    def __init__(self, stdout, stderr, exit_code, cmd, description=None,
+                 reason=None):
         """Init error and parent error class."""
-        if isinstance(cmd, (tuple, list)):
-            cmd = ' '.join(cmd)
         super(InTargetExecuteError, self).__init__(
-            stdout=stdout, stderr=stderr, exit_code=exit_code, cmd=cmd,
-            reason="Instance: {}".format(instance),
-            description=description if description else self.default_desc)
+            stdout=stdout, stderr=stderr, exit_code=exit_code,
+            cmd=shell_quote(cmd),
+            description=description if description else self.default_desc,
+            reason=reason)
 
 
 class TempDir(object):
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 6d621d26..275b16d2 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -18,6 +18,8 @@ from email.mime.application import MIMEApplication
 from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
 
+import httpretty
+
 from cloudinit import handlers
 from cloudinit import helpers as c_helpers
 from cloudinit import log
@@ -522,6 +524,54 @@ c: 4
         self.assertEqual(cfg.get('password'), 'gocubs')
         self.assertEqual(cfg.get('locale'), 'chicago')
 
+    @httpretty.activate
+    @mock.patch('cloudinit.url_helper.time.sleep')
+    def test_include(self, mock_sleep):
+        """Test #include."""
+        included_url = 'http://hostname/path'
+        included_data = '#cloud-config\nincluded: true\n'
+        httpretty.register_uri(httpretty.GET, included_url, included_data)
+
+        blob = '#include\n%s\n' % included_url
+
+        self.reRoot()
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(blob)
+        ci.fetch()
+        ci.consume_data()
+        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
+        cc = util.load_yaml(cc_contents)
+        self.assertTrue(cc.get('included'))
+
+    @httpretty.activate
+    @mock.patch('cloudinit.url_helper.time.sleep')
+    def test_include_bad_url(self, mock_sleep):
+        """Test #include with a bad URL."""
+        bad_url = 'http://bad/forbidden'
+        bad_data = '#cloud-config\nbad: true\n'
+        httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403)
+
+        included_url = 'http://hostname/path'
+        included_data = '#cloud-config\nincluded: true\n'
+        httpretty.register_uri(httpretty.GET, included_url, included_data)
+
+        blob = '#include\n%s\n%s' % (bad_url, included_url)
+
+        self.reRoot()
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(blob)
+        log_file = self.capture_log(logging.WARNING)
+        ci.fetch()
+        ci.consume_data()
+
+        self.assertIn("403 Client Error: Forbidden for url: %s" % bad_url,
+                      log_file.getvalue())
+
+        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
+        cc = util.load_yaml(cc_contents)
+        self.assertIsNone(cc.get('bad'))
+        self.assertTrue(cc.get('included'))
+
 
 class TestUDProcess(helpers.ResourceUsingTestCase):
 
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 6af699a6..ba328ee9 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -307,6 +307,39 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     @httpretty.activate
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp):
+        """Refresh the network_config Ec2 cache if network key is absent.
+
+        This catches an upgrade issue where obj.pkl contained stale metadata
+        which lacked newly required network key.
+        """
+        old_metadata = copy.deepcopy(DEFAULT_METADATA)
+        old_metadata.pop('network')
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md=old_metadata)
+        self.assertTrue(ds.get_data())
+        # Provide new revision of metadata that contains network data
+        register_mock_metaserver(
+            'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA)
+        mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
+        get_interface_mac_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        ds.fallback_nic = 'eth9'
+        with mock.patch(get_interface_mac_path) as m_get_interface_mac:
+            m_get_interface_mac.return_value = mac1
+            ds.network_config  # Will re-crawl network metadata
+        self.assertIn('Re-crawl of metadata service', self.logs.getvalue())
+        expected = {'version': 1, 'config': [
+            {'mac_address': '06:17:04:d7:26:09',
+             'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
+             'type': 'physical'}]}
+        self.assertEqual(expected, ds.network_config)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py
new file mode 100644
index 00000000..ced05a8d
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_etc_hosts.py
@@ -0,0 +1,69 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.config import cc_update_etc_hosts
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.tests import helpers as t_help
+
+import logging
+import os
+import shutil
+
+LOG = logging.getLogger(__name__)
+
+
+class TestHostsFile(t_help.FilesystemMockingTestCase):
+    def setUp(self):
+        super(TestHostsFile, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    def _fetch_distro(self, kind):
+        cls = distros.fetch(kind)
+        paths = helpers.Paths({})
+        return cls(kind, {}, paths)
+
+    def test_write_etc_hosts_suse_localhost(self):
+        cfg = {
+            'manage_etc_hosts': 'localhost',
+            'hostname': 'cloud-init.test.us'
+        }
+        os.makedirs('%s/etc/' % self.tmp)
+        hosts_content = '192.168.1.1 blah.blah.us blah\n'
+        fout = open('%s/etc/hosts' % self.tmp, 'w')
+        fout.write(hosts_content)
+        fout.close()
+        distro = self._fetch_distro('sles')
+        distro.hosts_fn = '%s/etc/hosts' % self.tmp
+        paths = helpers.Paths({})
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        cc_update_etc_hosts.handle('test', cfg, cc, LOG, [])
+        contents = util.load_file('%s/etc/hosts' % self.tmp)
+        if '127.0.0.1\tcloud-init.test.us\tcloud-init' not in contents:
+            self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
+        if '192.168.1.1\tblah.blah.us\tblah' not in contents:
+            self.assertIsNone('Default etc/hosts content modified')
+
+    def test_write_etc_hosts_suse_template(self):
+        cfg = {
+            'manage_etc_hosts': 'template',
+            'hostname': 'cloud-init.test.us'
+        }
+        shutil.copytree('templates', '%s/etc/cloud/templates' % self.tmp)
+        distro = self._fetch_distro('sles')
+        paths = helpers.Paths({})
+        paths.template_tpl = '%s' % self.tmp + '/etc/cloud/templates/%s.tmpl'
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        cc_update_etc_hosts.handle('test', cfg, cc, LOG, [])
+        contents = util.load_file('%s/etc/hosts' % self.tmp)
+        if '127.0.0.1 cloud-init.test.us cloud-init' not in contents:
+            self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
+        if '::1 cloud-init.test.us cloud-init' not in contents:
+            self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 3abe5786..28a8455d 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -430,5 +430,31 @@ class TestNtp(FilesystemMockingTestCase):
             "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n",
             content.decode())
 
+    def test_write_ntp_config_template_defaults_pools_empty_lists_sles(self):
+        """write_ntp_config_template defaults pools servers upon empty config.
+
+        When both pools and servers are empty, default NR_POOL_SERVERS get
+        configured.
+        """
+        distro = 'sles'
+        mycloud = self._get_cloud(distro)
+        ntp_conf = self.tmp_path('ntp.conf', self.new_root)  # Doesn't exist
+        # Create ntp.conf.tmpl
+        with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
+            stream.write(NTP_TEMPLATE)
+        with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
+            cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf)
+        content = util.read_file_or_url('file://' + ntp_conf).contents
+        default_pools = [
+            "{0}.opensuse.pool.ntp.org".format(x)
+            for x in range(0, cc_ntp.NR_POOL_SERVERS)]
+        self.assertEqual(
+            "servers []\npools {0}\n".format(default_pools),
+            content.decode())
+        self.assertIn(
+            "Adding distro default ntp pool servers: {0}".format(
+                ",".join(default_pools)),
+            self.logs.getvalue())
+
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index e9d5702a..22718108 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -2,6 +2,7 @@
 
 """Tests for registering RHEL subscription via rh_subscription."""
 
+import copy
 import logging
 
 from cloudinit.config import cc_rh_subscription
@@ -68,6 +69,20 @@ class GoodTests(TestCase):
         self.assertEqual(self.SM.log_success.call_count, 1)
         self.assertEqual(self.SM._sub_man_cli.call_count, 2)
 
+    @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos")
+    @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_sub_man_cli")
+    def test_update_repos_disable_with_none(self, m_sub_man_cli, m_get_repos):
+        cfg = copy.deepcopy(self.config)
+        m_get_repos.return_value = ([], ['repo1'])
+        m_sub_man_cli.return_value = (b'', b'')
+        cfg['rh_subscription'].update(
+            {'enable-repo': ['repo1'], 'disable-repo': None})
+        mysm = cc_rh_subscription.SubscriptionManager(cfg)
+        self.assertEqual(True, mysm.update_repos())
+        m_get_repos.assert_called_with()
+        self.assertEqual(m_sub_man_cli.call_args_list,
+                         [mock.call(['repos', '--enable=repo1'])])
+
     def test_full_registration(self):
         '''
         Registration with auto-attach, service-level, adding pools,